{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ede9183f-35c2-4e56-a894-18815cf36b8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4635f48b-ec8e-45a6-b903-da3332660569",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./CLUSTER_INFORMATION.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2b035b14-6a76-42da-8ffa-e199b43bd037",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cluster Name</th>\n",
       "      <th>Combination List</th>\n",
       "      <th>Cluster Centroid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Normal</td>\n",
       "      <td>['Regular', 'Typical', 'Common', 'Standard', '...</td>\n",
       "      <td>[0.04477231577038765, -1.1067426204681396, 0.6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Heartbleed</td>\n",
       "      <td>['SSL vulnerability', 'OpenSSL exploit', 'Info...</td>\n",
       "      <td>[0.20892228186130524, -1.8042747974395752, 2.1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Normal, DoS Hulk</td>\n",
       "      <td>['DDoS Shield', 'Steadyflow', 'Regular Traffic...</td>\n",
       "      <td>[-0.6773971915245056, 0.3524855971336365, -0.2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Normal, DDoS</td>\n",
       "      <td>['Coordinated Overload', 'Cyber Storm', 'Netwo...</td>\n",
       "      <td>[1.542575716972351, 1.337799310684204, -0.9742...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Exploits, SSH Patator</td>\n",
       "      <td>['Malicious SSH Exploiter', 'Unauthorized SSH ...</td>\n",
       "      <td>[0.8602568507194519, 0.048649415373802185, -0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511</th>\n",
       "      <td>DoS, FTP Patator</td>\n",
       "      <td>['BruteFTP', 'PayloadStorm', 'PassiveFlooder',...</td>\n",
       "      <td>[0.5987561941146851, -0.7863489985466003, -0.7...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512</th>\n",
       "      <td>DoS GoldenEye, Normal</td>\n",
       "      <td>['DestructiveStrike', 'PeakPerformance', 'Ruth...</td>\n",
       "      <td>[0.7857335209846497, 0.16938543319702148, -2.4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>513</th>\n",
       "      <td>SSH Patator</td>\n",
       "      <td>['SSH Brute Force', 'SSH Dictionary Attack', '...</td>\n",
       "      <td>[0.5677225589752197, 0.7569266557693481, -0.43...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>514</th>\n",
       "      <td>Fuzzers, Exploits, Normal</td>\n",
       "      <td>['Vulnerability Probing', 'Attack Simulation',...</td>\n",
       "      <td>[-1.5628975629806519, 1.0497198104858398, -0.8...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>DoS GoldenEye, DoS, Normal</td>\n",
       "      <td>['Overdrive Assault', 'Aggressive Storm', 'Chr...</td>\n",
       "      <td>[0.9662136435508728, -0.19642771780490875, -1....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>516 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Cluster Name  \\\n",
       "0                        Normal   \n",
       "1                    Heartbleed   \n",
       "2              Normal, DoS Hulk   \n",
       "3                  Normal, DDoS   \n",
       "4         Exploits, SSH Patator   \n",
       "..                          ...   \n",
       "511            DoS, FTP Patator   \n",
       "512       DoS GoldenEye, Normal   \n",
       "513                 SSH Patator   \n",
       "514   Fuzzers, Exploits, Normal   \n",
       "515  DoS GoldenEye, DoS, Normal   \n",
       "\n",
       "                                      Combination List  \\\n",
       "0    ['Regular', 'Typical', 'Common', 'Standard', '...   \n",
       "1    ['SSL vulnerability', 'OpenSSL exploit', 'Info...   \n",
       "2    ['DDoS Shield', 'Steadyflow', 'Regular Traffic...   \n",
       "3    ['Coordinated Overload', 'Cyber Storm', 'Netwo...   \n",
       "4    ['Malicious SSH Exploiter', 'Unauthorized SSH ...   \n",
       "..                                                 ...   \n",
       "511  ['BruteFTP', 'PayloadStorm', 'PassiveFlooder',...   \n",
       "512  ['DestructiveStrike', 'PeakPerformance', 'Ruth...   \n",
       "513  ['SSH Brute Force', 'SSH Dictionary Attack', '...   \n",
       "514  ['Vulnerability Probing', 'Attack Simulation',...   \n",
       "515  ['Overdrive Assault', 'Aggressive Storm', 'Chr...   \n",
       "\n",
       "                                      Cluster Centroid  \n",
       "0    [0.04477231577038765, -1.1067426204681396, 0.6...  \n",
       "1    [0.20892228186130524, -1.8042747974395752, 2.1...  \n",
       "2    [-0.6773971915245056, 0.3524855971336365, -0.2...  \n",
       "3    [1.542575716972351, 1.337799310684204, -0.9742...  \n",
       "4    [0.8602568507194519, 0.048649415373802185, -0....  \n",
       "..                                                 ...  \n",
       "511  [0.5987561941146851, -0.7863489985466003, -0.7...  \n",
       "512  [0.7857335209846497, 0.16938543319702148, -2.4...  \n",
       "513  [0.5677225589752197, 0.7569266557693481, -0.43...  \n",
       "514  [-1.5628975629806519, 1.0497198104858398, -0.8...  \n",
       "515  [0.9662136435508728, -0.19642771780490875, -1....  \n",
       "\n",
       "[516 rows x 3 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "31405ba8-021c-4b21-926b-ea994cd779de",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('./SCALER.pkl', 'rb') as f:\n",
    "    scaler = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "51839dbd-1ed2-4f09-9b9a-51eef458ae5e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoModel, AutoTokenizer, logging\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import warnings\n",
    "import docx\n",
    "import re\n",
    "import nltk\n",
    "import wikipedia\n",
    "import itertools\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "try:\n",
    "  ipython = get_ipython()\n",
    "  from tqdm.notebook import tqdm\n",
    "except:\n",
    "  from tqdm import tqdm\n",
    "\n",
    "logging.set_verbosity_error()\n",
    "logging.disable_progress_bar()\n",
    "warnings.filterwarnings('ignore')\n",
    "nltk.download('stopwords', quiet=True)\n",
    "nltk.download('punkt', quiet=True)\n",
    "nltk.download('wordnet', quiet=True)\n",
    "nltk.download('omw-1.4', quiet=True)\n",
    "nltk.download('tagsets', quiet=True)\n",
    "nltk.download('averaged_perceptron_tagger', quiet=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c162158f-1a6d-49a9-8138-4fc3bb5410fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BERTSimilarWords:\n",
    "\n",
    "    def __init__(self, model='bert-base-cased', max_heading_length = 10, max_document_length = 300, exclude_stopwords=[], embeddings_scaler=None):\n",
    "\n",
    "        for i in tqdm(range(2), unit=' it', desc='Initializing', postfix='Tokenizer and Model'): pass\n",
    "        self.tokenizer = AutoTokenizer.from_pretrained(model)\n",
    "        self.lemmatizer = WordNetLemmatizer()\n",
    "        self.min_max_scaler = MinMaxScaler()\n",
    "        self.scaler = embeddings_scaler\n",
    "        self.scaler_col_names = [str(i) for i in range(768)]\n",
    "        self.model = AutoModel.from_pretrained(model)\n",
    "        if torch.cuda.is_available():\n",
    "            self.processor = 'GPU'\n",
    "            self.cuda_current_device = torch.cuda.current_device()\n",
    "            self.model = self.model.to(self.cuda_current_device)\n",
    "        else:\n",
    "            self.processor = 'CPU'\n",
    "        self.max_document_length = max_document_length\n",
    "        self.max_heading_length = max_heading_length\n",
    "        self.max_ngram = 10\n",
    "        self.wikipedia_dataset_info = {}\n",
    "        self.document_list = []\n",
    "        self.bert_words = []\n",
    "        self.bert_vectors = []\n",
    "        self.bert_documents = []\n",
    "        self.continous_words = []\n",
    "        self.temporary_ngram_words = []\n",
    "        self.count_vectorizer_words = []\n",
    "        self.cv_counts = []\n",
    "        self.cv_words = []\n",
    "        self.count_vectorizer = CountVectorizer(analyzer=self._custom_analyzer)\n",
    "        self.stop_words = [word for word in stopwords.words() if word not in exclude_stopwords]\n",
    "        self.punctuations = '''!\"#$%&'()*+,-./:—;<=>−?–@[\\]^_`{|}~'''\n",
    "        self.doc_regex = \"[\\([][0-9]+[\\])]|[”“‘’‛‟]|\\d+\\s\"\n",
    "        self.punctuations_continuity_exclude = '''—-–,−'''\n",
    "        self.pos_tags_info = nltk.help.upenn_tagset\n",
    "        self.bert_words_ngram = [[] for _ in range(self.max_ngram)]\n",
    "        self.bert_vectors_ngram = [[] for _ in range(self.max_ngram)]\n",
    "        self.bert_documents_ngram = [[] for _ in range(self.max_ngram)]\n",
    "        self.bert_words_all = []\n",
    "        self.bert_vectors_all = []\n",
    "\n",
    "    def load_dataset(self, dataset_path=None, wikipedia_query=None, wikipedia_query_limit=10, wikipedia_page_list=None):\n",
    "\n",
    "        \"\"\"\n",
    "        This method extracts and processes the text content and generates word embeddings using the BERT model. Either one of the (dataset_path,wikipedia_query,wikipedia_page_list) parameters should be given.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        dataset_path : the dataset paths of the text files either as a string (one file) or a list of strings (multiple files) (supported files: .docx / .txt)\n",
    "\n",
    "        wikipedia_query : the Wikipedia search queries either as a string (one query) or a list of strings (multiple queries)\n",
    "\n",
    "        wikipedia_query_limit : maximum number of pages to extract for each query (only when wikipedia_query is given)\n",
    "\n",
    "        wikipedia_page_list : the list of names of Wikipedia pages to be extracted\n",
    "\n",
    "        \"\"\"\n",
    "\n",
    "        if wikipedia_query is not None or wikipedia_page_list is not None:\n",
    "            if wikipedia_query is not None:\n",
    "                query_results = []\n",
    "                if type(wikipedia_query) == str:\n",
    "                    wikipedia_query = [wikipedia_query]\n",
    "                for query in wikipedia_query:\n",
    "                    query_results += wikipedia.search(query, results=wikipedia_query_limit)\n",
    "            else:\n",
    "                query_results = wikipedia_page_list\n",
    "            page_content = []\n",
    "            for result in tqdm(query_results, unit=' pages', desc='Extracting', postfix='Data from Wikipedia'):\n",
    "                if '(disambiguation)' not in result and result not in self.wikipedia_dataset_info.keys():\n",
    "                    try:\n",
    "                        page = wikipedia.page(result, auto_suggest=False)\n",
    "                    except:\n",
    "                        continue\n",
    "                    page_content += ['== New page =='] + page.content.split('\\n\\n\\n')\n",
    "                    self.wikipedia_dataset_info[page.title] = page.url\n",
    "            self.document_list = self._process_wikipedia_dataset(page_content)\n",
    "        elif dataset_path is not None:\n",
    "            if type(dataset_path) == str:\n",
    "                dataset_path = [dataset_path]\n",
    "            for path in dataset_path:\n",
    "                if path.endswith('.docx'):\n",
    "                    docx_content = docx.Document(path)\n",
    "                    self.document_list += self._process_docx_dataset(docx_content)\n",
    "                elif path.endswith('.txt'):\n",
    "                    self.document_list += self._process_txt_dataset(path)\n",
    "                else:\n",
    "                    raise ValueError(\"Files supported: .docx / .txt\")\n",
    "        for words, vectors, document, continous in self._tokenize_and_embeddings(self.document_list):\n",
    "            self.temporary_ngram_words = []\n",
    "            for i in range(len(words)):\n",
    "                self._generate_n_grams(i, words, vectors, document, continous)\n",
    "            self.bert_words.extend(words)\n",
    "            self.bert_vectors.extend(vectors)\n",
    "            self.bert_documents.extend(document)\n",
    "            self.continous_words.extend(continous)\n",
    "            self.count_vectorizer_words.append(words + self.temporary_ngram_words)\n",
    "        self.bert_words_ngram[0] = self.bert_words\n",
    "        self.bert_vectors_ngram[0] = self.bert_vectors\n",
    "        self.bert_documents_ngram[0] = self.bert_documents\n",
    "        self.cv_counts = self.count_vectorizer.fit_transform(self.count_vectorizer_words)\n",
    "        self.cv_words = self.count_vectorizer.get_feature_names_out()\n",
    "        self.bert_words_all = np.array(list(itertools.chain.from_iterable(self.bert_words_ngram)))\n",
    "        self.bert_vectors_all = np.array(list(itertools.chain.from_iterable(self.bert_vectors_ngram)))\n",
    "        if self.scaler is not None:\n",
    "            df = pd.DataFrame(self.bert_vectors_all, columns=self.scaler_col_names)\n",
    "            self.bert_vectors_all = self.scaler.transform(df)\n",
    "            del df\n",
    "            for i in tqdm(range(self.max_ngram), desc='Producing', postfix='N-gram Words and Embeddings'):\n",
    "                df = pd.DataFrame(self.bert_vectors_ngram[i], columns=self.scaler_col_names)\n",
    "                self.bert_vectors_ngram[i] = self.scaler.transform(df)\n",
    "                del df\n",
    "        else:\n",
    "            for i in tqdm(range(self.max_ngram), desc='Producing', postfix='N-gram Words and Embeddings'):\n",
    "                self.bert_vectors_ngram[i] = np.array(self.bert_vectors_ngram[i])\n",
    "        return self\n",
    "\n",
    "    def _process_wikipedia_dataset(self, page_content):\n",
    "\n",
    "        document_list = []\n",
    "        for section in page_content:\n",
    "            if not any(exclude in section for exclude in\n",
    "                       ['== Further reading ==', '== References ==', '== External links ==', '== See also ==',\n",
    "                        '== Notes ==']):\n",
    "                if \"==\" in section[:self.max_heading_length] and \"===\" not in section[:self.max_heading_length]:\n",
    "                    flag = 0\n",
    "                paragraph = section.split('\\n')\n",
    "                for sentence in paragraph:\n",
    "                    sentence_words = sentence.split()\n",
    "                    sentence_length = len(sentence_words)\n",
    "                    if sentence_length > self.max_heading_length:\n",
    "                        if len(document_list) != 0 and flag == 1 and len(\n",
    "                                document_list[-1].split() + sentence_words) < self.max_document_length:\n",
    "                            document_list[-1] += ' ' + sentence\n",
    "                        else:\n",
    "                            document_list = self._process_dataset_long_paragraph(document_list, sentence,\n",
    "                                                                                 sentence_length)\n",
    "                            flag = 1\n",
    "        return document_list\n",
    "\n",
    "    def _process_docx_dataset(self, docx_content):\n",
    "\n",
    "        document_list = []\n",
    "        for paragraph in tqdm(docx_content.paragraphs, unit=' paragraphs', desc='Extracting',\n",
    "                              postfix='Data from Dataset'):\n",
    "            if 'Heading' in str(paragraph.style):\n",
    "                text = re.sub(self.doc_regex, '', paragraph.text)\n",
    "                if len(document_list) != 0 and len(document_list[-1].split()) <= self.max_heading_length:\n",
    "                    document_list[-1] = text + '.'\n",
    "                else:\n",
    "                    document_list.append(text + '.')\n",
    "            if 'Body Text' in str(paragraph.style):\n",
    "                sentence = re.sub(self.doc_regex, '', paragraph.text)\n",
    "                sentence_length = len(sentence.split())\n",
    "                if sentence_length > self.max_heading_length:\n",
    "                    if len(document_list) != 0 and len(\n",
    "                            document_list[-1].split()) + sentence_length < self.max_document_length:\n",
    "                        document_list[-1] += ' ' + sentence\n",
    "                    else:\n",
    "                        document_list = self._process_dataset_long_paragraph(document_list, sentence, sentence_length)\n",
    "        return document_list\n",
    "\n",
    "    def _process_txt_dataset(self, path):\n",
    "\n",
    "        document_list = []\n",
    "        with open(path) as file:\n",
    "            for line in tqdm(file.readlines(), unit=' paragraphs', desc='Extracting', postfix='Data from Dataset'):\n",
    "                line_text = line.strip()\n",
    "                line_text = re.sub(self.doc_regex, '', line_text)\n",
    "                line_length = len(line_text.split())\n",
    "                if 0 < line_length <= self.max_heading_length:\n",
    "                    if len(document_list) != 0 and len(document_list[-1].split()) <= self.max_heading_length:\n",
    "                        document_list[-1] = line_text + '.'\n",
    "                    else:\n",
    "                        document_list.append(line_text + '.')\n",
    "                elif line_length > self.max_heading_length:\n",
    "                    if len(document_list) != 0 and len(\n",
    "                            document_list[-1].split()) + line_length <= self.max_document_length:\n",
    "                        document_list[-1] += ' ' + line_text\n",
    "                    else:\n",
    "                        document_list = self._process_dataset_long_paragraph(document_list, line_text, len(\n",
    "                            document_list[-1].split()) + line_length)\n",
    "        return document_list\n",
    "\n",
    "    def _process_dataset_long_paragraph(self, document_list, sentence, sentence_length):\n",
    "\n",
    "        if sentence_length > self.max_document_length:\n",
    "            for i in range(2, sentence_length):\n",
    "                div = sentence_length / i\n",
    "                if div < self.max_document_length:\n",
    "                    break\n",
    "            temp_sent = ''\n",
    "            sm_sent = sent_tokenize(sentence)\n",
    "\n",
    "            for sent in sm_sent:\n",
    "                if len(temp_sent.split() + sent.split()) > div:\n",
    "                    if len(document_list[-1].split()) <= self.max_heading_length:\n",
    "                        document_list[-1] += ' ' + temp_sent\n",
    "                    else:\n",
    "                        document_list.append(temp_sent)\n",
    "                    temp_sent = ''\n",
    "                temp_sent = temp_sent + sent\n",
    "\n",
    "            if len(document_list[-1].split() + temp_sent.split()) < self.max_document_length:\n",
    "                document_list[-1] += ' ' + temp_sent\n",
    "            else:\n",
    "                document_list.append(temp_sent)\n",
    "        else:\n",
    "            document_list.append(sentence)\n",
    "        return document_list\n",
    "\n",
    "    def _tokenize_and_embeddings(self, document_list):\n",
    "\n",
    "        continous_index = 0\n",
    "        document_index = 0\n",
    "        for document in tqdm(document_list, unit=' documents', desc='Processing', postfix='Word Embeddings'):\n",
    "            if self.processor == 'GPU':\n",
    "                tokens = self.tokenizer(document, truncation=True, return_tensors='pt').to(self.cuda_current_device)\n",
    "            else:\n",
    "                tokens = self.tokenizer(document, truncation=True, return_tensors='pt')\n",
    "            words = self.tokenizer.convert_ids_to_tokens(tokens['input_ids'][0])\n",
    "            word_ids = tokens.word_ids()\n",
    "            output = self.model(**tokens)\n",
    "            if self.processor == 'GPU':\n",
    "                vectors = output.last_hidden_state[0].cpu().detach().numpy()\n",
    "            else:\n",
    "                vectors = output.last_hidden_state[0].detach().numpy()\n",
    "            word_list = []\n",
    "            vector_list = []\n",
    "            continous_words = []\n",
    "            word_index = -1\n",
    "            for i in range(len(words)):\n",
    "                if word_ids[i] is None or words[i] in self.punctuations:\n",
    "                    if words[i] in self.punctuations_continuity_exclude:\n",
    "                        pass\n",
    "                    else:\n",
    "                        continous_index = continous_index + 1\n",
    "                    continue\n",
    "                if word_ids[i] > word_index:\n",
    "                    if len(word_list) != 0 and word_list[-1].lower() in self.stop_words:\n",
    "                        word_list.pop()\n",
    "                        vector_list.pop()\n",
    "                        continous_words.pop()\n",
    "                        continous_index = continous_index + 1\n",
    "                    word_list.append(words[i])\n",
    "                    vector_list.append(vectors[i])\n",
    "                    continous_words.append(continous_index)\n",
    "                    word_index = word_ids[i]\n",
    "                elif word_ids[i] == word_index:\n",
    "                    sub_word = words[i].replace('##', \"\")\n",
    "                    word_list[-1] = word_list[-1] + sub_word\n",
    "                    vector_list[-1] = (vector_list[-1] + vectors[i])\n",
    "                    if word_ids[i + 1] != word_ids[i]:\n",
    "                        vector_list[-1] = vector_list[-1] / word_ids.count(word_index)\n",
    "            yield word_list, vector_list, [document_index] * len(word_list), continous_words\n",
    "            document_index += 1\n",
    "\n",
    "    def _generate_n_grams(self, i, words, vectors, document, continous, n=1):\n",
    "\n",
    "        if i > n - 1 and n < self.max_ngram and continous[i] == continous[i - n]:\n",
    "            temp_word = ''\n",
    "            temp_vector = np.zeros([len(vectors[i])])\n",
    "            for j in range(n, -1, -1):\n",
    "                temp_word = temp_word + ' ' + words[i - j]\n",
    "                temp_vector = temp_vector + vectors[i - j]\n",
    "            self.temporary_ngram_words.append(temp_word.strip())\n",
    "            self.bert_words_ngram[n].append(temp_word.strip())\n",
    "            self.bert_vectors_ngram[n].append(temp_vector / (n + 1))\n",
    "            self.bert_documents_ngram[n].append(document[i])\n",
    "            self._generate_n_grams(i, words, vectors, document, continous, n=n + 1)\n",
    "        return\n",
    "\n",
    "    def _custom_analyzer(self, words):\n",
    "\n",
    "        final_list = []\n",
    "        for word in words:\n",
    "            final_list.append(word)\n",
    "            lemmatized_word = ' '.join([self.lemmatizer.lemmatize(token.lower()) for token in word.split()])\n",
    "            if word != lemmatized_word:\n",
    "                final_list.append(lemmatized_word)\n",
    "        return final_list\n",
    "\n",
    "    def _context_similarity_measurement(self, features, context_length):\n",
    "\n",
    "        context_total = 0\n",
    "        word_total = 0\n",
    "        for i in range(context_length):\n",
    "            if features[i] != 0:\n",
    "                context_total += 1\n",
    "        for i, x in enumerate(features[context_length:]):\n",
    "            if x != 0:\n",
    "                word_total += 1\n",
    "        word_mean = 0.5 * np.mean(features[context_length:])\n",
    "        if len(features[:context_length]) == 0:\n",
    "            context_mean = 0\n",
    "        else:\n",
    "            context_mean = 0.5 * np.mean(features[:context_length])\n",
    "        return int(str(context_total) + str(word_total)) + context_mean + word_mean\n",
    "\n",
    "    def _get_article_words_vectors(self, similar_documents, similarity_scores, similarity_factor, input_words_max):\n",
    "\n",
    "        document_words = []\n",
    "        document_vectors = np.empty((0, similar.bert_vectors[0].shape[0]))\n",
    "        for article in similar_documents:\n",
    "            if similarity_scores[article] < similarity_scores[similar_documents[0]] - similarity_factor:\n",
    "                break\n",
    "            if article == len(similar_documents) - 1:\n",
    "                for i in range(input_words_max):\n",
    "                    document_words += self.bert_words_ngram[i][self.bert_documents_ngram[i].index(article):]\n",
    "                    document_vectors = np.append(document_vectors, self.bert_vectors_ngram[i][self.bert_documents_ngram[i].index(article):], axis=0)\n",
    "            else:\n",
    "                for i in range(input_words_max):\n",
    "                    document_words += self.bert_words_ngram[i][\n",
    "                                      self.bert_documents_ngram[i].index(article):self.bert_documents_ngram[i].index(\n",
    "                                          article + 1)]\n",
    "                    document_vectors = np.append(document_vectors, self.bert_vectors_ngram[i][\n",
    "                                        self.bert_documents_ngram[i].index(article):self.bert_documents_ngram[i].index(\n",
    "                                            article + 1)], axis=0)\n",
    "        return document_words, document_vectors\n",
    "\n",
    "    def _calculate_input_word_embedding(self, input_words, document_words, document_vectors, uncased_lemmatization):\n",
    "\n",
    "        average_list = np.zeros([len(input_words), len(document_vectors[0])])\n",
    "        mean_index = []\n",
    "        for i_index, i_word in enumerate(input_words):\n",
    "            a_count = 0\n",
    "            for a_index, a_word in enumerate(document_words):\n",
    "                if uncased_lemmatization and i_word == self.lemmatizer.lemmatize(a_word.lower()):\n",
    "                    average_list[i_index] += document_vectors[a_index]\n",
    "                    a_count = a_count + 1\n",
    "                elif i_word == a_word:\n",
    "                    average_list[i_index] += document_vectors[a_index]\n",
    "                    a_count = a_count + 1\n",
    "            if average_list[i_index].any():\n",
    "                average_list[i_index] = average_list[i_index] / a_count\n",
    "                mean_index.append(i_index)\n",
    "        average = np.mean(average_list[mean_index], axis=0)\n",
    "        return average\n",
    "\n",
    "    def _context_similarity_document_scores(self, input_context_words, input_context_length, input_words_length,\n",
    "                                            context_similarity_factor):\n",
    "\n",
    "        cv_list = []\n",
    "        cv_counts = self.cv_counts.toarray()\n",
    "        index = [i for i in np.searchsorted(self.cv_words, input_context_words) if\n",
    "                 self.cv_words[i] in input_context_words]\n",
    "\n",
    "        for i in range(len(self.document_list)):\n",
    "            cv_list.append(cv_counts[i][index].tolist())\n",
    "\n",
    "        cv_list = self.min_max_scaler.fit_transform(cv_list)\n",
    "        similarity_scores = [self._context_similarity_measurement(counts, input_context_length) for counts in cv_list]\n",
    "        similarity_factor = context_similarity_factor * input_words_length\n",
    "        similar_documents = np.flip(np.argsort(similarity_scores))\n",
    "        return similar_documents, similarity_scores, similarity_factor\n",
    "\n",
    "    def _find_nearest_cosine_words(self, input_context_words, cosine_sim, cosine_words, pos_to_exclude,\n",
    "                                   max_output_words, output_filter_factor):\n",
    "\n",
    "        output_dict = {}\n",
    "        sorted_list = np.flip(np.argsort(cosine_sim))\n",
    "        lemmatized_words = {self.lemmatizer.lemmatize(token.lower()) for word in input_context_words for token in\n",
    "                            word.split()}\n",
    "\n",
    "        for i in range(len(cosine_words)):\n",
    "            stop = 0\n",
    "            pop_list = []\n",
    "            original_word = cosine_words[sorted_list[i]]\n",
    "            pos_tags = [pos[1] for pos in nltk.pos_tag(original_word.split())]\n",
    "            lemmatized_word = {self.lemmatizer.lemmatize(token.lower()) for token in original_word.split()}\n",
    "            if len(lemmatized_words.intersection(lemmatized_word)) > output_filter_factor * len(original_word.split()):\n",
    "                continue\n",
    "            if any(pos in pos_tags for pos in pos_to_exclude):\n",
    "                continue\n",
    "            if original_word not in output_dict.keys():\n",
    "                for word in output_dict.keys():\n",
    "                    if original_word in word:\n",
    "                        stop = 1\n",
    "                        break\n",
    "                    elif word in original_word:\n",
    "                        pop_list.append(word)\n",
    "                        stop = 0\n",
    "                if stop == 0:\n",
    "                    pop = [output_dict.pop(key) for key in pop_list]\n",
    "                    output_dict[original_word] = cosine_sim[sorted_list[i]]\n",
    "                    if len(output_dict.keys()) == max_output_words:\n",
    "                        break\n",
    "        return output_dict\n",
    "\n",
    "    def _process_input_context_words(self, input_context, input_words, single_word_split, uncased_lemmatization):\n",
    "\n",
    "        if single_word_split:\n",
    "            input_context_split = input_context.split()\n",
    "            input_words_split = list(itertools.chain.from_iterable([word.split() for word in input_words]))\n",
    "            input_words_max = 1\n",
    "        else:\n",
    "            input_context_split = [] if input_context == '' else [input_context]\n",
    "            input_words_split = input_words\n",
    "            input_words_max = max([len(word.split()) for word in input_words])\n",
    "        if uncased_lemmatization:\n",
    "            input_context_split = [' '.join([self.lemmatizer.lemmatize(token.lower()) for token in word.split()]) for\n",
    "                                   word in input_context_split]\n",
    "            input_words_split = [' '.join([self.lemmatizer.lemmatize(token.lower()) for token in word.split()]) for word\n",
    "                                 in input_words_split]\n",
    "        input_context_words = input_context_split + input_words_split\n",
    "        input_context_words_max = max([len(word.split()) for word in input_context_words])\n",
    "        return input_context_split, input_words_split, input_words_max, input_context_words, input_context_words_max\n",
    "\n",
    "    def find_similar_words(self,\n",
    "                           input_context='',\n",
    "                           input_words=[],\n",
    "                           input_embedding = np.array([]),\n",
    "                           output_words_ngram=1,\n",
    "                           pos_to_exclude=[],\n",
    "                           max_output_words=10,\n",
    "                           context_similarity_factor=0.25,\n",
    "                           output_filter_factor=0.5,\n",
    "                           single_word_split=True,\n",
    "                           uncased_lemmatization=True\n",
    "                           ):\n",
    "        \"\"\"\n",
    "        This method calculates the cosine similarity between the average of the input words based on the given context and all the words present in the given vocabulary.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        input_context : the input context (string) (optional) (default: None)\n",
    "\n",
    "        input_words : the input words as (list of strings)\n",
    "\n",
    "        output_words_ngram : n-gram words expected as output (integer) (optional) (default: 1)\n",
    "\n",
    "        pos_to_exclude : the words are ignored in the output if these part of speech tags are present in it (list of strings) (optional) (default: None)\n",
    "\n",
    "        max_output_words : the maximum number of output words to be generated (integer) (optional) (default: 10)\n",
    "\n",
    "        context_similarity_factor : uses to tune the context-matching process, find the best paragraphs related to the given input words (float) (optional) (default: 0.25) (Range: 0 to 1)\n",
    "\n",
    "        output_filter_factor : uses to ignore words that are similar to the given input in the output (float) (optional) (default: 0.5) (Range: 0 to 1)\n",
    "\n",
    "        single_word_split : whether to split n-gram words when given as input (boolean) (optional) (default: True)\n",
    "\n",
    "        uncased_lemmatization : whether to uncase and lemmatize the input (boolean) (optional) (default: True)\n",
    "\n",
    "        \"\"\"\n",
    "\n",
    "        if input_embedding.size == 0:\n",
    "\n",
    "            input_context_split, input_words_split, input_words_max, input_context_words, input_context_words_max = self._process_input_context_words(\n",
    "                input_context, input_words, single_word_split, uncased_lemmatization)\n",
    "\n",
    "            similar_documents, similarity_scores, similarity_factor = self._context_similarity_document_scores(\n",
    "                input_context_words, len(input_context_split), len(input_words_split), context_similarity_factor)\n",
    "\n",
    "            document_words, document_vectors = self._get_article_words_vectors(similar_documents, similarity_scores,\n",
    "                                                                              similarity_factor, input_words_max)\n",
    "\n",
    "            input_embedding = self._calculate_input_word_embedding(input_words_split, document_words, document_vectors,\n",
    "                                                                  uncased_lemmatization)\n",
    "        else:\n",
    "            input_context_words = []\n",
    "\n",
    "        if output_words_ngram == 0:\n",
    "            cosine_sim = cosine_similarity(self.bert_vectors_all,\n",
    "                                           [input_embedding]).flatten()\n",
    "            cosine_words = self.bert_words_all\n",
    "        else:\n",
    "            cosine_sim = cosine_similarity(self.bert_vectors_ngram[output_words_ngram - 1], [input_embedding]).flatten()\n",
    "            cosine_words = self.bert_words_ngram[output_words_ngram - 1]\n",
    "\n",
    "        output_dictionary = self._find_nearest_cosine_words(input_context_words, cosine_sim, cosine_words,\n",
    "                                                            pos_to_exclude, max_output_words, output_filter_factor)\n",
    "        return output_dictionary, input_embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "556c3ba6-5da7-438d-8b26-159a2d8c5e00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03e8ec5b0c9140a680675e661cec9e21",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Initializing:   0%|          | 0/2 [00:00<?, ? it/s, Tokenizer and Model]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "828ab3a24feb418bbf9f1415b1ff5a00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Extracting:   0%|          | 0/6020 [00:00<?, ? paragraphs/s, Data from Dataset]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bca5482038a24c0a9f1967167850051b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Processing:   0%|          | 0/3010 [00:00<?, ? documents/s, Word Embeddings]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b477b2985cc418e8143804f0a7a5505",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating:   0%|          | 0/10 [00:00<?, ?it/s, N-gram Words]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "similar = BERTSimilarWords(model='rdpahalavan/bert-network-packet-flow-header-payload', max_document_length=375, exclude_stopwords=['dos'], embeddings_scaler=scaler).load_dataset(dataset_path='CORPUS.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3dacc54f-2e53-4c8d-894c-743c46e6f1cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "476013"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(similar.bert_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b9a65143-a8db-471a-b4e6-4815d76d0afc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "868533"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(similar.bert_words_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "02fbd841-dd83-47f0-bcf4-43d292d91a7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "476013"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(similar.bert_words_ngram[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "46d531a2-d4cd-4c1f-9777-297df9a7f926",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1-gram words: 476013\n",
      "2-gram words: 224565\n",
      "3-gram words: 94640\n",
      "4-gram words: 39059\n",
      "5-gram words: 17326\n",
      "6-gram words: 8469\n",
      "7-gram words: 4313\n",
      "8-gram words: 2284\n",
      "9-gram words: 1218\n",
      "10-gram words: 646\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(f'{i+1}-gram words: {len(similar.bert_words_ngram[i])}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1042beec-f2d4-4301-8979-0470334e86d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cluster Name</th>\n",
       "      <th>Combination List</th>\n",
       "      <th>Cluster Centroid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Normal</td>\n",
       "      <td>['Regular', 'Typical', 'Common', 'Standard', '...</td>\n",
       "      <td>[0.04477231577038765, -1.1067426204681396, 0.6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Heartbleed</td>\n",
       "      <td>['SSL vulnerability', 'OpenSSL exploit', 'Info...</td>\n",
       "      <td>[0.20892228186130524, -1.8042747974395752, 2.1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Normal, DoS Hulk</td>\n",
       "      <td>['DDoS Shield', 'Steadyflow', 'Regular Traffic...</td>\n",
       "      <td>[-0.6773971915245056, 0.3524855971336365, -0.2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Normal, DDoS</td>\n",
       "      <td>['Coordinated Overload', 'Cyber Storm', 'Netwo...</td>\n",
       "      <td>[1.542575716972351, 1.337799310684204, -0.9742...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Exploits, SSH Patator</td>\n",
       "      <td>['Malicious SSH Exploiter', 'Unauthorized SSH ...</td>\n",
       "      <td>[0.8602568507194519, 0.048649415373802185, -0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511</th>\n",
       "      <td>DoS, FTP Patator</td>\n",
       "      <td>['BruteFTP', 'PayloadStorm', 'PassiveFlooder',...</td>\n",
       "      <td>[0.5987561941146851, -0.7863489985466003, -0.7...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512</th>\n",
       "      <td>DoS GoldenEye, Normal</td>\n",
       "      <td>['DestructiveStrike', 'PeakPerformance', 'Ruth...</td>\n",
       "      <td>[0.7857335209846497, 0.16938543319702148, -2.4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>513</th>\n",
       "      <td>SSH Patator</td>\n",
       "      <td>['SSH Brute Force', 'SSH Dictionary Attack', '...</td>\n",
       "      <td>[0.5677225589752197, 0.7569266557693481, -0.43...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>514</th>\n",
       "      <td>Fuzzers, Exploits, Normal</td>\n",
       "      <td>['Vulnerability Probing', 'Attack Simulation',...</td>\n",
       "      <td>[-1.5628975629806519, 1.0497198104858398, -0.8...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>DoS GoldenEye, DoS, Normal</td>\n",
       "      <td>['Overdrive Assault', 'Aggressive Storm', 'Chr...</td>\n",
       "      <td>[0.9662136435508728, -0.19642771780490875, -1....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>516 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Cluster Name  \\\n",
       "0                        Normal   \n",
       "1                    Heartbleed   \n",
       "2              Normal, DoS Hulk   \n",
       "3                  Normal, DDoS   \n",
       "4         Exploits, SSH Patator   \n",
       "..                          ...   \n",
       "511            DoS, FTP Patator   \n",
       "512       DoS GoldenEye, Normal   \n",
       "513                 SSH Patator   \n",
       "514   Fuzzers, Exploits, Normal   \n",
       "515  DoS GoldenEye, DoS, Normal   \n",
       "\n",
       "                                      Combination List  \\\n",
       "0    ['Regular', 'Typical', 'Common', 'Standard', '...   \n",
       "1    ['SSL vulnerability', 'OpenSSL exploit', 'Info...   \n",
       "2    ['DDoS Shield', 'Steadyflow', 'Regular Traffic...   \n",
       "3    ['Coordinated Overload', 'Cyber Storm', 'Netwo...   \n",
       "4    ['Malicious SSH Exploiter', 'Unauthorized SSH ...   \n",
       "..                                                 ...   \n",
       "511  ['BruteFTP', 'PayloadStorm', 'PassiveFlooder',...   \n",
       "512  ['DestructiveStrike', 'PeakPerformance', 'Ruth...   \n",
       "513  ['SSH Brute Force', 'SSH Dictionary Attack', '...   \n",
       "514  ['Vulnerability Probing', 'Attack Simulation',...   \n",
       "515  ['Overdrive Assault', 'Aggressive Storm', 'Chr...   \n",
       "\n",
       "                                      Cluster Centroid  \n",
       "0    [0.04477231577038765, -1.1067426204681396, 0.6...  \n",
       "1    [0.20892228186130524, -1.8042747974395752, 2.1...  \n",
       "2    [-0.6773971915245056, 0.3524855971336365, -0.2...  \n",
       "3    [1.542575716972351, 1.337799310684204, -0.9742...  \n",
       "4    [0.8602568507194519, 0.048649415373802185, -0....  \n",
       "..                                                 ...  \n",
       "511  [0.5987561941146851, -0.7863489985466003, -0.7...  \n",
       "512  [0.7857335209846497, 0.16938543319702148, -2.4...  \n",
       "513  [0.5677225589752197, 0.7569266557693481, -0.43...  \n",
       "514  [-1.5628975629806519, 1.0497198104858398, -0.8...  \n",
       "515  [0.9662136435508728, -0.19642771780490875, -1....  \n",
       "\n",
       "[516 rows x 3 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c29366d6-d097-4366-9518-87eefdcd730d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"['Regular', 'Typical', 'Common', 'Standard', 'Regularized']\""
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Combination List'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8663e7bd-89d4-49b6-b771-c17f90cfc297",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"['Regular', 'Standard', 'Routine', 'Common', 'Typical']\""
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Combination List'][500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e8cfa19e-2ac2-4bb7-a80b-c4a7e61b9a4d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accepted behavior patterns observed': 0.9800301522430844,\n",
       " 'Standard behavior': 0.9775606474812201,\n",
       " 'signifies': 0.9773024906893971,\n",
       " 'differentiate regular traffic': 0.9757654693549477,\n",
       " 'vulnerabilities': 0.9756843923825949,\n",
       " 'routine network activities including regular communication data transfer': 0.9750090825589134,\n",
       " 'typical': 0.9741505065735123,\n",
       " 'malicious intent': 0.9732447395603571,\n",
       " 'conforming': 0.9718956874685923,\n",
       " 'unexpected behavior helping': 0.970450529274012}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags, emb = similar.find_similar_words(input_words=eval(df['Combination List'][500]), context_similarity_factor=0, output_words_ngram=0, uncased_lemmatization=True, single_word_split=False, output_filter_factor=1)\n",
    "tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3eeba8f3-a960-44c7-9875-f6c40ddb7435",
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_embeddings = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd8b1de0-8601-443b-9650-e935f76f2a6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(df)):\n",
    "    try:\n",
    "        tags, emb = similar.find_similar_words(input_words=eval(df['Combination List'][i]), context_similarity_factor=0, output_words_ngram=0, uncased_lemmatization=True, single_word_split=False, output_filter_factor=1)\n",
    "    except:\n",
    "        tags, emb = similar.find_similar_words(input_words=eval(df['Combination List'][i]), context_similarity_factor=0, output_words_ngram=0, uncased_lemmatization=True, single_word_split=True, output_filter_factor=1)\n",
    "    tag_embeddings.append(emb)\n",
    "    print(f'Completed: {i}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "62636a45-3dca-4b8f-84f1-1cc3e7cc6ac5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "516"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tag_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "dc1148e8-d23a-40ce-a7b1-77479ed3e8e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('./TAGS-NAMES-EMBEDDINGS.npy', tag_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6a2a44e9-071a-4a33-a25c-daaaf53d29a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster_centers = np.load('./KMEANS-CLUSTER-CENTERS.npy')\n",
    "tag_names = np.load('./TAGS-NAMES-EMBEDDINGS.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "519a98b0-bb36-4737-b4d3-94aceb2eb6ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "main_df = pd.read_csv('./DATA_TRAIN.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "32febf8c-16fa-4729-a189-8206b0dad359",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packet_dat</th>\n",
       "      <th>attack_cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0 0 141 -1 80 63713 2960 2920 64 0 5 0 -1 119 ...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1190 1582 3526815 -1 80 50095 1500 1460 118 0 ...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0 0 4 -1 80 41471 4420 4380 64 0 5 0 -1 72 84 ...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0 0 176 -1 80 45284 2948 2896 64 0 8 0 -1 72 8...</td>\n",
       "      <td>DoS Hulk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0 0 128 -1 80 46654 1500 1448 64 0 8 0 -1 72 8...</td>\n",
       "      <td>DoS Hulk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187776</th>\n",
       "      <td>14492 14492 0 -1 51328 22 164 112 62 0 8 3 -1 ...</td>\n",
       "      <td>SSH Patator</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187777</th>\n",
       "      <td>14 98 131788 -1 80 52067 1500 1460 253 0 5 0 -...</td>\n",
       "      <td>DoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187778</th>\n",
       "      <td>1 2 397 -1 47188 22 692 640 62 0 8 3 -1 0 0 2 ...</td>\n",
       "      <td>SSH Patator</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187779</th>\n",
       "      <td>2063 0 0 -1 80 32768 1500 1448 64 0 8 0 -1 32 ...</td>\n",
       "      <td>DoS Hulk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187780</th>\n",
       "      <td>1 1 34 -1 56628 21 78 26 62 0 8 3 -1 80 65 83 ...</td>\n",
       "      <td>FTP Patator</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1187781 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                packet_dat   attack_cat\n",
       "0        0 0 141 -1 80 63713 2960 2920 64 0 5 0 -1 119 ...         DDoS\n",
       "1        1190 1582 3526815 -1 80 50095 1500 1460 118 0 ...       Normal\n",
       "2        0 0 4 -1 80 41471 4420 4380 64 0 5 0 -1 72 84 ...         DDoS\n",
       "3        0 0 176 -1 80 45284 2948 2896 64 0 8 0 -1 72 8...     DoS Hulk\n",
       "4        0 0 128 -1 80 46654 1500 1448 64 0 8 0 -1 72 8...     DoS Hulk\n",
       "...                                                    ...          ...\n",
       "1187776  14492 14492 0 -1 51328 22 164 112 62 0 8 3 -1 ...  SSH Patator\n",
       "1187777  14 98 131788 -1 80 52067 1500 1460 253 0 5 0 -...          DoS\n",
       "1187778  1 2 397 -1 47188 22 692 640 62 0 8 3 -1 0 0 2 ...  SSH Patator\n",
       "1187779  2063 0 0 -1 80 32768 1500 1448 64 0 8 0 -1 32 ...     DoS Hulk\n",
       "1187780  1 1 34 -1 56628 21 78 26 62 0 8 3 -1 80 65 83 ...  FTP Patator\n",
       "\n",
       "[1187781 rows x 2 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "314d2560-b6f2-4e9d-9726-c21a577b6e2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, AutoModel\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"rdpahalavan/bert-network-packet-flow-header-payload\")\n",
    "\n",
    "model = AutoModel.from_pretrained(\"rdpahalavan/bert-network-packet-flow-header-payload\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "61018be4-6852-4377-9a88-233da9ddbc42",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model.to(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "4a684c97-f15a-4f9a-acef-484c3d813b1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler_col_names = [str(i) for i in range(768)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "591b385d-541f-4423-88fd-d4fc2a217cf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import euclidean_distances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c619df7e-4cd4-4345-8c9b-b19c04f08cb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_packet_embedding(packet):\n",
    "    tokens = tokenizer(packet, truncation=True, return_tensors='pt').to(0)\n",
    "    output = model(**tokens)\n",
    "    embedding = output.last_hidden_state[:, 1:-1, :].mean(dim=1).cpu().detach().numpy()\n",
    "    df = pd.DataFrame(embedding[0].reshape(1, -1), columns=scaler_col_names)\n",
    "    embedding = scaler.transform(df)[0]\n",
    "    euclidean_distance = euclidean_distances(cluster_centers, [embedding])\n",
    "    data = {i: euclidean_distance[i][0] for i in range(len(euclidean_distance))}\n",
    "    df = pd.DataFrame(list(data.items()), columns=['Cluster', 'Euclidean Distance'])\n",
    "    df.sort_values(by='Euclidean Distance', inplace=True)\n",
    "    return int(df.iloc[0]['Cluster']), df.iloc[0]['Euclidean Distance'], embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a23fdfcf-4dd0-42a8-9dcf-530f064e371d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Cluster Name                                   DoS, Exploits, Generic\n",
       "Combination List    ['Vulnerability Bomb', 'Overwhelming Assault',...\n",
       "Cluster Centroid    [-0.8283041715621948, -0.6053891777992249, -0....\n",
       "Name: 178, dtype: object"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[178]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "878cc1fc-86f2-4639-8bb1-c7df94adbc0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packet_dat</th>\n",
       "      <th>attack_cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0 0 141 -1 80 63713 2960 2920 64 0 5 0 -1 119 ...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1190 1582 3526815 -1 80 50095 1500 1460 118 0 ...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0 0 4 -1 80 41471 4420 4380 64 0 5 0 -1 72 84 ...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0 0 176 -1 80 45284 2948 2896 64 0 8 0 -1 72 8...</td>\n",
       "      <td>DoS Hulk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0 0 128 -1 80 46654 1500 1448 64 0 8 0 -1 72 8...</td>\n",
       "      <td>DoS Hulk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187776</th>\n",
       "      <td>14492 14492 0 -1 51328 22 164 112 62 0 8 3 -1 ...</td>\n",
       "      <td>SSH Patator</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187777</th>\n",
       "      <td>14 98 131788 -1 80 52067 1500 1460 253 0 5 0 -...</td>\n",
       "      <td>DoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187778</th>\n",
       "      <td>1 2 397 -1 47188 22 692 640 62 0 8 3 -1 0 0 2 ...</td>\n",
       "      <td>SSH Patator</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187779</th>\n",
       "      <td>2063 0 0 -1 80 32768 1500 1448 64 0 8 0 -1 32 ...</td>\n",
       "      <td>DoS Hulk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187780</th>\n",
       "      <td>1 1 34 -1 56628 21 78 26 62 0 8 3 -1 80 65 83 ...</td>\n",
       "      <td>FTP Patator</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1187781 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                packet_dat   attack_cat\n",
       "0        0 0 141 -1 80 63713 2960 2920 64 0 5 0 -1 119 ...         DDoS\n",
       "1        1190 1582 3526815 -1 80 50095 1500 1460 118 0 ...       Normal\n",
       "2        0 0 4 -1 80 41471 4420 4380 64 0 5 0 -1 72 84 ...         DDoS\n",
       "3        0 0 176 -1 80 45284 2948 2896 64 0 8 0 -1 72 8...     DoS Hulk\n",
       "4        0 0 128 -1 80 46654 1500 1448 64 0 8 0 -1 72 8...     DoS Hulk\n",
       "...                                                    ...          ...\n",
       "1187776  14492 14492 0 -1 51328 22 164 112 62 0 8 3 -1 ...  SSH Patator\n",
       "1187777  14 98 131788 -1 80 52067 1500 1460 253 0 5 0 -...          DoS\n",
       "1187778  1 2 397 -1 47188 22 692 640 62 0 8 3 -1 0 0 2 ...  SSH Patator\n",
       "1187779  2063 0 0 -1 80 32768 1500 1448 64 0 8 0 -1 32 ...     DoS Hulk\n",
       "1187780  1 1 34 -1 56628 21 78 26 62 0 8 3 -1 80 65 83 ...  FTP Patator\n",
       "\n",
       "[1187781 rows x 2 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "a0f2adee-9cdb-4fe7-8682-12d476aee7cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster, distance, embedding = get_packet_embedding(main_df['packet_dat'][953456])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "4b48d4fc-5db1-465f-92fa-f08d0350aa12",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "63"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "55ab2782-9321-4d00-996d-1a4fdeaf84d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.968107223510742"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "00ce3fd5-dbea-4223-b53f-eebe9d770fe1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Cluster Name                                              SSH Patator\n",
       "Combination List    ['SSH Password Cracker', 'Brute Force SSH Atta...\n",
       "Cluster Centroid    [0.6212525963783264, 0.7067753672599792, -0.76...\n",
       "Name: 63, dtype: object"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[63]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "a71c6161-8956-4c16-9ea3-84057ac2321a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'SSH Credential Guessing': 0.955725148046819,\n",
       " 'SSH Password Guessing': 0.9526428861127418,\n",
       " 'Secure Shell': 0.9485803671888409,\n",
       " 'SSH Dictionary based Attack': 0.9485583280322055,\n",
       " 'secure remote access': 0.9458850748318011,\n",
       " 'SSH Login Attack': 0.9451930643133879,\n",
       " 'unauthorized access': 0.9451043138417081,\n",
       " 'SSH authentication attack': 0.94485077834196,\n",
       " 'successful login attempts': 0.9435012262817355,\n",
       " 'SSH Dictionary Attacker': 0.9430031735410134}"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags, emb = similar.find_similar_words(input_embedding = (tag_names[cluster] + (embedding - cluster_centers[cluster])), context_similarity_factor=0, output_words_ngram=0, uncased_lemmatization=True, single_word_split=False, output_filter_factor=1)\n",
    "tags"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37f1cccb-40d6-461e-a3b3-648c7ad484aa",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "6db2a151-728f-49aa-abb8-9df5556b468c",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('./DATA_TEST.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "9f0e7fdc-b1c7-4729-af96-c54233dce2f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packet_dat</th>\n",
       "      <th>attack_cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0 0 2 -1 80 56471 10175 10135 64 0 5 3 -1 112 ...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>111 16 147912 -1 38156 25 1500 1460 254 0 5 0 ...</td>\n",
       "      <td>DoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>14 11 1332 -1 13447 111 100 60 254 0 5 0 -1 12...</td>\n",
       "      <td>Reconnaissance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>22 14 63411 -1 444 45022 8740 8688 64 0 8 0 -1...</td>\n",
       "      <td>Heartbleed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>157 213 474513 -1 443 54320 1500 1460 53 0 5 0...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>122 18 163802 -1 4664 25 1500 1460 254 0 5 0 -...</td>\n",
       "      <td>DoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2 1 3347 -1 80 33580 471 419 64 0 8 3 -1 219 1...</td>\n",
       "      <td>Web Attack - XSS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>127 298 495757 -1 80 22701 2960 2920 64 0 5 0 ...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>8130 4065 56910 -1 21 59698 72 20 64 0 8 3 -1 ...</td>\n",
       "      <td>FTP Patator</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0 0 4 -1 80 50564 2875 2835 64 0 5 2 -1 109 97...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>155 335 497999 -1 80 48724 1500 1448 30 0 8 3 ...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>14 8 2073 -1 41985 49800 402 362 255 0 5 0 -1 ...</td>\n",
       "      <td>Exploits</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>5 3 3389 -1 80 60312 1101 1049 64 0 8 3 -1 72 ...</td>\n",
       "      <td>Web Attack - Brute Force</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1 0 39 -1 443 49256 1500 1448 55 0 8 0 -1 70 2...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>13 88 118281 -1 143 53385 1500 1460 253 0 5 0 ...</td>\n",
       "      <td>DoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>41 0 248 -1 80 63794 5880 5840 64 0 5 0 -1 72 ...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>699 937 2084435 -1 80 50128 1500 1460 119 0 5 ...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>1 2 331 -1 22 50766 116 64 64 0 8 3 -1 158 175...</td>\n",
       "      <td>SSH Patator</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>23 17 66510 -1 444 45022 4396 4344 64 0 8 0 -1...</td>\n",
       "      <td>Heartbleed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1 2 397 -1 22 52328 132 80 64 0 8 3 -1 137 169...</td>\n",
       "      <td>SSH Patator</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           packet_dat  \\\n",
       "1   0 0 2 -1 80 56471 10175 10135 64 0 5 3 -1 112 ...   \n",
       "4   111 16 147912 -1 38156 25 1500 1460 254 0 5 0 ...   \n",
       "5   14 11 1332 -1 13447 111 100 60 254 0 5 0 -1 12...   \n",
       "6   22 14 63411 -1 444 45022 8740 8688 64 0 8 0 -1...   \n",
       "12  157 213 474513 -1 443 54320 1500 1460 53 0 5 0...   \n",
       "15  122 18 163802 -1 4664 25 1500 1460 254 0 5 0 -...   \n",
       "16  2 1 3347 -1 80 33580 471 419 64 0 8 3 -1 219 1...   \n",
       "17  127 298 495757 -1 80 22701 2960 2920 64 0 5 0 ...   \n",
       "18  8130 4065 56910 -1 21 59698 72 20 64 0 8 3 -1 ...   \n",
       "21  0 0 4 -1 80 50564 2875 2835 64 0 5 2 -1 109 97...   \n",
       "22  155 335 497999 -1 80 48724 1500 1448 30 0 8 3 ...   \n",
       "23  14 8 2073 -1 41985 49800 402 362 255 0 5 0 -1 ...   \n",
       "27  5 3 3389 -1 80 60312 1101 1049 64 0 8 3 -1 72 ...   \n",
       "28  1 0 39 -1 443 49256 1500 1448 55 0 8 0 -1 70 2...   \n",
       "29  13 88 118281 -1 143 53385 1500 1460 253 0 5 0 ...   \n",
       "30  41 0 248 -1 80 63794 5880 5840 64 0 5 0 -1 72 ...   \n",
       "32  699 937 2084435 -1 80 50128 1500 1460 119 0 5 ...   \n",
       "34  1 2 331 -1 22 50766 116 64 64 0 8 3 -1 158 175...   \n",
       "35  23 17 66510 -1 444 45022 4396 4344 64 0 8 0 -1...   \n",
       "36  1 2 397 -1 22 52328 132 80 64 0 8 3 -1 137 169...   \n",
       "\n",
       "                  attack_cat  \n",
       "1                       DDoS  \n",
       "4                        DoS  \n",
       "5             Reconnaissance  \n",
       "6                 Heartbleed  \n",
       "12                    Normal  \n",
       "15                       DoS  \n",
       "16          Web Attack - XSS  \n",
       "17                      DDoS  \n",
       "18               FTP Patator  \n",
       "21                      DDoS  \n",
       "22                    Normal  \n",
       "23                  Exploits  \n",
       "27  Web Attack - Brute Force  \n",
       "28                    Normal  \n",
       "29                       DoS  \n",
       "30                      DDoS  \n",
       "32                    Normal  \n",
       "34               SSH Patator  \n",
       "35                Heartbleed  \n",
       "36               SSH Patator  "
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df[test_df['attack_cat']!='DoS Hulk'].head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "2b7f5d51-7ba8-4967-93c3-d315748299e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster, distance, embedding = get_packet_embedding(test_df['packet_dat'][29969])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "2f12b54e-5ae5-4488-9795-29d16bdc6605",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "packet_dat    31 31 15599 -1 54131 444 343 303 128 0 5 3 -1 ...\n",
       "attack_cat                                         Infiltration\n",
       "Name: 29969, dtype: object"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.loc[29969]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "7f35054e-9b48-4b05-9a17-b81953386e4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17, 1.8301955461502075)"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster, distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "3198158f-f0a5-47cf-ae5c-1f1c7d1aba52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'routers firewalls': 0.9642805725907245,\n",
       " 'subversive entry': 0.9623438670437829,\n",
       " 'malicious payloads': 0.9617977693854897,\n",
       " 'address software vulnerabilities robust network monitoring': 0.961637982183049,\n",
       " 'IP spoofing session hijacking': 0.9603539512293087,\n",
       " 'configuration perform regular security audits': 0.9602574015344214,\n",
       " 'strict access': 0.9599809415814653,\n",
       " 'packets reach': 0.9592730400260735,\n",
       " 'safe online': 0.9591345643930638,\n",
       " 'intrusion detection systems': 0.9591310571446596}"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags, emb = similar.find_similar_words(input_embedding = (tag_names[cluster] + (embedding - cluster_centers[cluster])), context_similarity_factor=0, output_words_ngram=0, uncased_lemmatization=True, single_word_split=False, output_filter_factor=1)\n",
    "tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "5f38a2e0-41ba-439b-87cd-ea88932533aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = test_df.groupby('attack_cat').apply(lambda x: x.sample(min(len(x), 2500))).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "fe6815c9-10b5-4425-a01c-ed223d88d81f",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = test_df.sample(frac=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "03a1c4f1-9442-4ca5-8f6c-e60a6933a66d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packet_dat</th>\n",
       "      <th>attack_cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>29969</th>\n",
       "      <td>31 31 15599 -1 54131 444 343 303 128 0 5 3 -1 ...</td>\n",
       "      <td>Infiltration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26900</th>\n",
       "      <td>125 19 167141 -1 7977 25 1500 1460 254 0 5 0 -...</td>\n",
       "      <td>Generic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35952</th>\n",
       "      <td>53 31 5932 -1 80 34338 1500 1460 252 0 5 0 -1 ...</td>\n",
       "      <td>Reconnaissance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3801</th>\n",
       "      <td>0 0 1 -1 80 56417 4335 4295 64 0 5 3 -1 114 11...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>541</th>\n",
       "      <td>19 15 3868 -1 42812 80 239 199 62 0 5 3 -1 71 ...</td>\n",
       "      <td>Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43585</th>\n",
       "      <td>7326 0 0 -1 80 59042 1919 1867 64 0 8 3 -1 72 ...</td>\n",
       "      <td>Web Attack - XSS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45875</th>\n",
       "      <td>14 77 103183 -1 80 49608 1500 1460 253 0 5 0 -...</td>\n",
       "      <td>Worms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30893</th>\n",
       "      <td>15 15 4167 -1 54119 444 165 125 128 0 5 3 -1 3...</td>\n",
       "      <td>Infiltration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46303</th>\n",
       "      <td>14 77 103183 -1 80 49608 1500 1460 252 0 5 0 -...</td>\n",
       "      <td>Worms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3436</th>\n",
       "      <td>0 0 128 -1 80 56145 7255 7215 64 0 5 2 -1 32 3...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>46781 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              packet_dat        attack_cat\n",
       "29969  31 31 15599 -1 54131 444 343 303 128 0 5 3 -1 ...      Infiltration\n",
       "26900  125 19 167141 -1 7977 25 1500 1460 254 0 5 0 -...           Generic\n",
       "35952  53 31 5932 -1 80 34338 1500 1460 252 0 5 0 -1 ...    Reconnaissance\n",
       "3801   0 0 1 -1 80 56417 4335 4295 64 0 5 3 -1 114 11...              DDoS\n",
       "541    19 15 3868 -1 42812 80 239 199 62 0 5 3 -1 71 ...          Analysis\n",
       "...                                                  ...               ...\n",
       "43585  7326 0 0 -1 80 59042 1919 1867 64 0 8 3 -1 72 ...  Web Attack - XSS\n",
       "45875  14 77 103183 -1 80 49608 1500 1460 253 0 5 0 -...             Worms\n",
       "30893  15 15 4167 -1 54119 444 165 125 128 0 5 3 -1 3...      Infiltration\n",
       "46303  14 77 103183 -1 80 49608 1500 1460 252 0 5 0 -...             Worms\n",
       "3436   0 0 128 -1 80 56145 7255 7215 64 0 5 2 -1 32 3...              DDoS\n",
       "\n",
       "[46781 rows x 2 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "30b79489-9ca3-4da1-898a-99c93e38aab0",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df.to_csv('tag_generation.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "ee90b7e7-af32-4b95-8fb7-f44254daa242",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_testing_list = []\n",
    "index_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79535e3c-c499-40fe-837f-0b187487c531",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, packet in enumerate(test_df['packet_dat'].to_list()):\n",
    "    if i > -1:\n",
    "        cluster, distance, embedding = get_packet_embedding(test_df['packet_dat'][i])\n",
    "        tags, emb = similar.find_similar_words(input_embedding = (tag_names[cluster] + (embedding - cluster_centers[cluster])), context_similarity_factor=0, output_words_ngram=0, uncased_lemmatization=True, single_word_split=False, output_filter_factor=1)\n",
    "        tags_testing_list.append(tags)\n",
    "        index_list.append(i)\n",
    "        if i%1000 == 0:\n",
    "            temp = pd.DataFrame({'df_index': index_list, 'tags': tags_testing_list})\n",
    "            temp.to_csv(f'./Tags Testing/Tags{i}.csv', index=False)\n",
    "        print(f'Completed: {i}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "id": "a5065fae-0528-410b-ad57-cf9f3be53269",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'11 71 94466 -1 80 40371 1500 1460 253 0 5 0 -1 15 136 177 4 0 0 131 125 8 12 15 143 167 4 0 0 139 117 12 133 246 15 132 39 5 0 0 139 85 8 185 0 48 11 8 139 69 12 139 92 145 64 137 4 36 137 92 36 4 232 41 99 1 0 133 192 137 218 117 10 141 101 244 137 208 91 94 95 93 195 199 4 36 229 221 9 8 49 201 49 219 137 157 96 255 255 255 137 141 100 255 255 255 232 205 94 0 0 133 192 116 9 128 56 0 15 133 119 4 0 0 131 125 8 6 15 132 71 1 0 0 139 93 12 184 0 0 0 0 133 192 137 157 92 255 255 255 116 12 199 4 36 92 83 11 8 232 184 193 249 247 139 149 100 255 255 255 141 181 92 255 255 255 139 133 96 255 255 255 137 52 36 139 77 8 232 219 6 0 0 131 236 4 137 195 133 192 15 132 245 0 0 0 131 120 24 255 116 7 199 64 24 255 255 255 255 139 133 92 255 255 255 61 117 160 10 8 116 22 137 4 36 232 236 98 1 0 137 133 92 255 255 255 133 192 15 132 197 0 0 0 137 116 36 4 139 69 8 137 4 36 232 255 4 0 0 133 192 137 199 15 132 151 0 0 0 139 85 8 190 0 48 11 8 139 4 149 128 222 9 8 137 28 149 0 48 11 8 133 192 117 121 139 77 8 139 157 92 255 255 255 139 68 142 64 57 216 116 22 61 117 160 10 8 116 8 137 4 36 232 91 28 255 255 139 69 8 137 92 134 64 161 88 48 11 8 57 248 116 21 61 117 160 10 8 116 8 137 4 36 232 60 28 255 255 137 61 88 48 11 8 255 5 196 83 11 8 184 0 0 0 0 133 192 116 12 199 4 36 92 83 11 8 232 203 192 249 247 139 133 96 255 255 255 137 4 36 232 13 28 255 255 139 149 92 255 255 255 233 159 254 255 255 255 208 235 131 139 133 92 255 255 255 61 117 160 10 8 116 8 137 4 36 232 233 27 255 255 49 192 137 133 92 255 255 255 235 175 199 69 8 0 0 0 0 144 141 180 38 0 0 0 0 131 125 8 6 116 10 139 77 12 139 85 8 137 76 149 168 255 69 8 131 125 8 12 126 231 199 69 8 13 0 0 0 139 93 12 191 59'"
      ]
     },
     "execution_count": 265,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df['packet_dat'][index_list[-1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "id": "d164db28-8513-40b2-b254-6b1edf2ba82c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46780"
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index_list[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "id": "a6bad42a-305e-4a12-a504-40c5305c097a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "packet_dat    11 71 94466 -1 80 40371 1500 1460 253 0 5 0 -1...\n",
       "attack_cat                                                Worms\n",
       "Name: 46780, dtype: object"
      ]
     },
     "execution_count": 269,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.loc[46780]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "303c74e7-379b-4215-baa2-b89a0082b09c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46781"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tags_testing_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "9e5d091d-6bb7-4aa6-a44d-ad89e72beb27",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'id':index_list, 'tags':tags_testing_list})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "442bb026-f15b-4ae7-9821-cbd9d64797b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>{'ZeroImpact': 0.9326767845982844, 'injecting'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>{'injecting unauthorized packets': 0.929202050...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>{'unknowingly triggering': 0.9352225230817839,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>{'Exploit DoS': 0.9215557725519854, 'mitigate ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>{'ZeroImpact': 0.9545252141232174, 'anomalies'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46776</th>\n",
       "      <td>46776</td>\n",
       "      <td>{'successful Exploit Normalizer attack': 0.940...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46777</th>\n",
       "      <td>46777</td>\n",
       "      <td>{'misconfigured firewalls weak authentication ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46778</th>\n",
       "      <td>46778</td>\n",
       "      <td>{'malicious code': 0.9816162270729385, 'infect...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46779</th>\n",
       "      <td>46779</td>\n",
       "      <td>{'Vulnerability Hunter': 0.9274233934166645, '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46780</th>\n",
       "      <td>46780</td>\n",
       "      <td>{'misconfigured firewalls weak authentication ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>46781 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                               tags\n",
       "0          0  {'ZeroImpact': 0.9326767845982844, 'injecting'...\n",
       "1          1  {'injecting unauthorized packets': 0.929202050...\n",
       "2          2  {'unknowingly triggering': 0.9352225230817839,...\n",
       "3          3  {'Exploit DoS': 0.9215557725519854, 'mitigate ...\n",
       "4          4  {'ZeroImpact': 0.9545252141232174, 'anomalies'...\n",
       "...      ...                                                ...\n",
       "46776  46776  {'successful Exploit Normalizer attack': 0.940...\n",
       "46777  46777  {'misconfigured firewalls weak authentication ...\n",
       "46778  46778  {'malicious code': 0.9816162270729385, 'infect...\n",
       "46779  46779  {'Vulnerability Hunter': 0.9274233934166645, '...\n",
       "46780  46780  {'misconfigured firewalls weak authentication ...\n",
       "\n",
       "[46781 rows x 2 columns]"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "fb9622a0-2531-4883-9062-7fceceb9de3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('GENERATED_TAGS.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "1ff96a4e-131a-43fc-96c8-24a987acec9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packet_dat</th>\n",
       "      <th>attack_cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>29969</th>\n",
       "      <td>31 31 15599 -1 54131 444 343 303 128 0 5 3 -1 ...</td>\n",
       "      <td>Infiltration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26900</th>\n",
       "      <td>125 19 167141 -1 7977 25 1500 1460 254 0 5 0 -...</td>\n",
       "      <td>Generic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35952</th>\n",
       "      <td>53 31 5932 -1 80 34338 1500 1460 252 0 5 0 -1 ...</td>\n",
       "      <td>Reconnaissance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3801</th>\n",
       "      <td>0 0 1 -1 80 56417 4335 4295 64 0 5 3 -1 114 11...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>541</th>\n",
       "      <td>19 15 3868 -1 42812 80 239 199 62 0 5 3 -1 71 ...</td>\n",
       "      <td>Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43585</th>\n",
       "      <td>7326 0 0 -1 80 59042 1919 1867 64 0 8 3 -1 72 ...</td>\n",
       "      <td>Web Attack - XSS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45875</th>\n",
       "      <td>14 77 103183 -1 80 49608 1500 1460 253 0 5 0 -...</td>\n",
       "      <td>Worms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30893</th>\n",
       "      <td>15 15 4167 -1 54119 444 165 125 128 0 5 3 -1 3...</td>\n",
       "      <td>Infiltration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46303</th>\n",
       "      <td>14 77 103183 -1 80 49608 1500 1460 252 0 5 0 -...</td>\n",
       "      <td>Worms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3436</th>\n",
       "      <td>0 0 128 -1 80 56145 7255 7215 64 0 5 2 -1 32 3...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>46781 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              packet_dat        attack_cat\n",
       "29969  31 31 15599 -1 54131 444 343 303 128 0 5 3 -1 ...      Infiltration\n",
       "26900  125 19 167141 -1 7977 25 1500 1460 254 0 5 0 -...           Generic\n",
       "35952  53 31 5932 -1 80 34338 1500 1460 252 0 5 0 -1 ...    Reconnaissance\n",
       "3801   0 0 1 -1 80 56417 4335 4295 64 0 5 3 -1 114 11...              DDoS\n",
       "541    19 15 3868 -1 42812 80 239 199 62 0 5 3 -1 71 ...          Analysis\n",
       "...                                                  ...               ...\n",
       "43585  7326 0 0 -1 80 59042 1919 1867 64 0 8 3 -1 72 ...  Web Attack - XSS\n",
       "45875  14 77 103183 -1 80 49608 1500 1460 253 0 5 0 -...             Worms\n",
       "30893  15 15 4167 -1 54119 444 165 125 128 0 5 3 -1 3...      Infiltration\n",
       "46303  14 77 103183 -1 80 49608 1500 1460 252 0 5 0 -...             Worms\n",
       "3436   0 0 128 -1 80 56145 7255 7215 64 0 5 2 -1 32 3...              DDoS\n",
       "\n",
       "[46781 rows x 2 columns]"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8ad7dec5-60ac-43e4-b964-2f787961f3e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./GENERATED_TAGS.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "96b0c31d-8408-40bf-ace8-f5cdee3dec72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>{'ZeroImpact': 0.9326767845982844, 'injecting'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>{'injecting unauthorized packets': 0.929202050...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>{'unknowingly triggering': 0.9352225230817839,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>{'Exploit DoS': 0.9215557725519854, 'mitigate ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>{'ZeroImpact': 0.9545252141232174, 'anomalies'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46776</th>\n",
       "      <td>46776</td>\n",
       "      <td>{'successful Exploit Normalizer attack': 0.940...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46777</th>\n",
       "      <td>46777</td>\n",
       "      <td>{'misconfigured firewalls weak authentication ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46778</th>\n",
       "      <td>46778</td>\n",
       "      <td>{'malicious code': 0.9816162270729385, 'infect...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46779</th>\n",
       "      <td>46779</td>\n",
       "      <td>{'Vulnerability Hunter': 0.9274233934166645, '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46780</th>\n",
       "      <td>46780</td>\n",
       "      <td>{'misconfigured firewalls weak authentication ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>46781 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                               tags\n",
       "0          0  {'ZeroImpact': 0.9326767845982844, 'injecting'...\n",
       "1          1  {'injecting unauthorized packets': 0.929202050...\n",
       "2          2  {'unknowingly triggering': 0.9352225230817839,...\n",
       "3          3  {'Exploit DoS': 0.9215557725519854, 'mitigate ...\n",
       "4          4  {'ZeroImpact': 0.9545252141232174, 'anomalies'...\n",
       "...      ...                                                ...\n",
       "46776  46776  {'successful Exploit Normalizer attack': 0.940...\n",
       "46777  46777  {'misconfigured firewalls weak authentication ...\n",
       "46778  46778  {'malicious code': 0.9816162270729385, 'infect...\n",
       "46779  46779  {'Vulnerability Hunter': 0.9274233934166645, '...\n",
       "46780  46780  {'misconfigured firewalls weak authentication ...\n",
       "\n",
       "[46781 rows x 2 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "49d2607d-74c0-4a4e-aff8-35ada18fd48a",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df2 = pd.read_csv('./tag_generation.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "f87a5294-267a-4377-93a9-e616f7e518cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packet_dat</th>\n",
       "      <th>attack_cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>31 31 15599 -1 54131 444 343 303 128 0 5 3 -1 ...</td>\n",
       "      <td>Infiltration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>125 19 167141 -1 7977 25 1500 1460 254 0 5 0 -...</td>\n",
       "      <td>Generic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>53 31 5932 -1 80 34338 1500 1460 252 0 5 0 -1 ...</td>\n",
       "      <td>Reconnaissance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0 0 1 -1 80 56417 4335 4295 64 0 5 3 -1 114 11...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>19 15 3868 -1 42812 80 239 199 62 0 5 3 -1 71 ...</td>\n",
       "      <td>Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46776</th>\n",
       "      <td>7326 0 0 -1 80 59042 1919 1867 64 0 8 3 -1 72 ...</td>\n",
       "      <td>Web Attack - XSS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46777</th>\n",
       "      <td>14 77 103183 -1 80 49608 1500 1460 253 0 5 0 -...</td>\n",
       "      <td>Worms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46778</th>\n",
       "      <td>15 15 4167 -1 54119 444 165 125 128 0 5 3 -1 3...</td>\n",
       "      <td>Infiltration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46779</th>\n",
       "      <td>14 77 103183 -1 80 49608 1500 1460 252 0 5 0 -...</td>\n",
       "      <td>Worms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46780</th>\n",
       "      <td>0 0 128 -1 80 56145 7255 7215 64 0 5 2 -1 32 3...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>46781 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              packet_dat        attack_cat\n",
       "0      31 31 15599 -1 54131 444 343 303 128 0 5 3 -1 ...      Infiltration\n",
       "1      125 19 167141 -1 7977 25 1500 1460 254 0 5 0 -...           Generic\n",
       "2      53 31 5932 -1 80 34338 1500 1460 252 0 5 0 -1 ...    Reconnaissance\n",
       "3      0 0 1 -1 80 56417 4335 4295 64 0 5 3 -1 114 11...              DDoS\n",
       "4      19 15 3868 -1 42812 80 239 199 62 0 5 3 -1 71 ...          Analysis\n",
       "...                                                  ...               ...\n",
       "46776  7326 0 0 -1 80 59042 1919 1867 64 0 8 3 -1 72 ...  Web Attack - XSS\n",
       "46777  14 77 103183 -1 80 49608 1500 1460 253 0 5 0 -...             Worms\n",
       "46778  15 15 4167 -1 54119 444 165 125 128 0 5 3 -1 3...      Infiltration\n",
       "46779  14 77 103183 -1 80 49608 1500 1460 252 0 5 0 -...             Worms\n",
       "46780  0 0 128 -1 80 56145 7255 7215 64 0 5 2 -1 32 3...              DDoS\n",
       "\n",
       "[46781 rows x 2 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "cd871082-a8dc-41ce-9fb4-4b69b2a7ea96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Infiltration                  2500\n",
       "Heartbleed                    2500\n",
       "Worms                         2500\n",
       "Fuzzers                       2500\n",
       "FTP Patator                   2500\n",
       "Web Attack - Brute Force      2500\n",
       "DoS GoldenEye                 2500\n",
       "Generic                       2500\n",
       "SSH Patator                   2500\n",
       "DoS                           2500\n",
       "Normal                        2500\n",
       "DoS Hulk                      2500\n",
       "Exploits                      2500\n",
       "DDoS                          2500\n",
       "Reconnaissance                2500\n",
       "DoS SlowHTTPTest              2432\n",
       "DoS Slowloris                 1852\n",
       "Web Attack - XSS              1549\n",
       "Bot                           1460\n",
       "Analysis                       793\n",
       "Backdoor                       507\n",
       "Shellcode                      435\n",
       "Port Scan                      238\n",
       "Web Attack - SQL Injection      15\n",
       "Name: attack_cat, dtype: int64"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df2['attack_cat'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "4f4d31df-830a-4a4d-806b-7d8cc929b485",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                                                      0\n",
       "tags    {'ZeroImpact': 0.9326767845982844, 'injecting'...\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "c4ec6cf2-bb39-4f0f-988a-c5f21b27e0bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Analysis'"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df['attack_cat'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "id": "4db827b9-8e76-40ac-8899-d33e433b6865",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                                                  46780\n",
       "tags    {'misconfigured firewalls weak authentication ...\n",
       "Name: 46780, dtype: object"
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[46780]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "id": "dfc048cd-f268-4ba2-a2d0-ef6e3c0c4a30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'misconfigured firewalls weak authentication mechanisms, unauthorized, PersistentIntrusion attack, Malware Launcher, StealthySilencer, EncryptionBreach, exploits vulnerabilities, VulnerabilityChain, Packet spoofing, Vulnerability Scanner'"
      ]
     },
     "execution_count": 281,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "', '.join(df.loc[46780, 'tags'].keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "id": "b22e24d9-9ce2-4419-96ce-493122aad9cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ZeroImpact', 'injecting', 'responsiveness']"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(df.loc[0, 'tags'].keys())[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "id": "53876919-bd2f-4398-94ff-e126c8b443a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ZeroImpact,injecting,responsiveness']"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[','.join(list(df.loc[0, 'tags'].keys())[:3])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "fbfc1392-c29f-415a-9abd-24aa9c9feab6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer, util"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "id": "0defb81a-02a9-464b-867b-0f7d66aee2cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Analysis']"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[test_df['attack_cat'][0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "id": "52af6c74-c8f4-4996-b8a5-354deebce3a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "index = 46780\n",
    "\n",
    "sentences = [f\"{test_df['attack_cat'][index]}, network packet\"] + list(df.loc[index, 'tags'].keys()) + [', '.join(df.loc[index, 'tags'].keys())] + [','.join(list(df.loc[index, 'tags'].keys())[:3])] + [','.join(list(df.loc[index, 'tags'].keys())[:5])] + [','.join(list(df.loc[index, 'tags'].keys())[6:])]\n",
    "\n",
    "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n",
    "embeddings = model.encode(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "id": "f5157b5b-0482-41d1-8452-bc656688b148",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Worms, network packet',\n",
       " 'misconfigured firewalls weak authentication mechanisms',\n",
       " 'unauthorized',\n",
       " 'PersistentIntrusion attack',\n",
       " 'Malware Launcher',\n",
       " 'StealthySilencer',\n",
       " 'EncryptionBreach',\n",
       " 'exploits vulnerabilities',\n",
       " 'VulnerabilityChain',\n",
       " 'Packet spoofing',\n",
       " 'Vulnerability Scanner',\n",
       " 'misconfigured firewalls weak authentication mechanisms, unauthorized, PersistentIntrusion attack, Malware Launcher, StealthySilencer, EncryptionBreach, exploits vulnerabilities, VulnerabilityChain, Packet spoofing, Vulnerability Scanner',\n",
       " 'misconfigured firewalls weak authentication mechanisms,unauthorized,PersistentIntrusion attack',\n",
       " 'misconfigured firewalls weak authentication mechanisms,unauthorized,PersistentIntrusion attack,Malware Launcher,StealthySilencer',\n",
       " 'exploits vulnerabilities,VulnerabilityChain,Packet spoofing,Vulnerability Scanner']"
      ]
     },
     "execution_count": 282,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "id": "e373cc06-a732-4b74-81a0-6ee6608a583a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packet_dat</th>\n",
       "      <th>attack_cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>42728</th>\n",
       "      <td>3937 1968 0 -1 80 36196 582 530 64 0 8 3 -1 72...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42719</th>\n",
       "      <td>0 0 523 -1 80 36214 2073 2021 64 0 8 3 -1 72 8...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42730</th>\n",
       "      <td>12820 12820 0 -1 36204 80 651 599 62 0 8 3 -1 ...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42717</th>\n",
       "      <td>1 0 947 -1 80 36212 1500 1448 64 0 8 0 -1 72 8...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42720</th>\n",
       "      <td>0 0 523 -1 36204 80 651 599 62 0 8 3 -1 71 69 ...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42724</th>\n",
       "      <td>0 0 210 -1 80 36206 582 530 64 0 8 3 -1 72 84 ...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42729</th>\n",
       "      <td>0 0 933 -1 80 36202 4201 4149 64 0 8 3 -1 72 8...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42718</th>\n",
       "      <td>0 0 195 -1 80 36196 582 530 64 0 8 3 -1 72 84 ...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42725</th>\n",
       "      <td>13698 13698 0 -1 36242 80 651 599 62 0 8 3 -1 ...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42721</th>\n",
       "      <td>14084 14084 0 -1 36206 80 575 523 62 0 8 3 -1 ...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42731</th>\n",
       "      <td>23255 23255 0 -1 80 36200 1933 1881 64 0 8 3 -...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42726</th>\n",
       "      <td>0 0 946 -1 36242 80 651 599 62 0 8 3 -1 71 69 ...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42723</th>\n",
       "      <td>23809 23809 0 -1 36198 80 512 460 62 0 8 3 -1 ...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42722</th>\n",
       "      <td>0 0 523 -1 80 36244 2073 2021 64 0 8 3 -1 72 8...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42727</th>\n",
       "      <td>9174 9174 0 -1 36244 80 651 599 62 0 8 3 -1 71...</td>\n",
       "      <td>Web Attack - SQL Injection</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              packet_dat  \\\n",
       "42728  3937 1968 0 -1 80 36196 582 530 64 0 8 3 -1 72...   \n",
       "42719  0 0 523 -1 80 36214 2073 2021 64 0 8 3 -1 72 8...   \n",
       "42730  12820 12820 0 -1 36204 80 651 599 62 0 8 3 -1 ...   \n",
       "42717  1 0 947 -1 80 36212 1500 1448 64 0 8 0 -1 72 8...   \n",
       "42720  0 0 523 -1 36204 80 651 599 62 0 8 3 -1 71 69 ...   \n",
       "42724  0 0 210 -1 80 36206 582 530 64 0 8 3 -1 72 84 ...   \n",
       "42729  0 0 933 -1 80 36202 4201 4149 64 0 8 3 -1 72 8...   \n",
       "42718  0 0 195 -1 80 36196 582 530 64 0 8 3 -1 72 84 ...   \n",
       "42725  13698 13698 0 -1 36242 80 651 599 62 0 8 3 -1 ...   \n",
       "42721  14084 14084 0 -1 36206 80 575 523 62 0 8 3 -1 ...   \n",
       "42731  23255 23255 0 -1 80 36200 1933 1881 64 0 8 3 -...   \n",
       "42726  0 0 946 -1 36242 80 651 599 62 0 8 3 -1 71 69 ...   \n",
       "42723  23809 23809 0 -1 36198 80 512 460 62 0 8 3 -1 ...   \n",
       "42722  0 0 523 -1 80 36244 2073 2021 64 0 8 3 -1 72 8...   \n",
       "42727  9174 9174 0 -1 36244 80 651 599 62 0 8 3 -1 71...   \n",
       "\n",
       "                       attack_cat  \n",
       "42728  Web Attack - SQL Injection  \n",
       "42719  Web Attack - SQL Injection  \n",
       "42730  Web Attack - SQL Injection  \n",
       "42717  Web Attack - SQL Injection  \n",
       "42720  Web Attack - SQL Injection  \n",
       "42724  Web Attack - SQL Injection  \n",
       "42729  Web Attack - SQL Injection  \n",
       "42718  Web Attack - SQL Injection  \n",
       "42725  Web Attack - SQL Injection  \n",
       "42721  Web Attack - SQL Injection  \n",
       "42731  Web Attack - SQL Injection  \n",
       "42726  Web Attack - SQL Injection  \n",
       "42723  Web Attack - SQL Injection  \n",
       "42722  Web Attack - SQL Injection  \n",
       "42727  Web Attack - SQL Injection  "
      ]
     },
     "execution_count": 298,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df[test_df['attack_cat']=='Web Attack - SQL Injection']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "id": "0a50d145-a259-4982-ad6a-ac0a878737cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 0\n",
    "\n",
    "sentences = [f\"The network packet attack label is {test_df['attack_cat'][i]}\"] + list(df.loc[i, 'tags'].keys()) + [', '.join(df.loc[i, 'tags'].keys())] + [', '.join(list(df.loc[i, 'tags'].keys())[:3])] + [', '.join(list(df.loc[i, 'tags'].keys())[:5])] + [', '.join(list(df.loc[i, 'tags'].keys())[5:])]\n",
    "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n",
    "embeddings = model.encode(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "id": "411b04b0-aaeb-44b9-8a6f-96ab0958d6e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.0762, 0.0879, 0.0472, 0.2492, 0.3381, 0.0877, 0.0468, 0.0061, 0.3375,\n",
      "        0.4865, 0.4895, 0.0506, 0.3742, 0.3952])\n",
      "['The network packet attack label is Analysis', 'ZeroImpact', 'injecting', 'responsiveness', 'spoofing', 'data exfiltration', 'encrypted', 'reassembly', 'reordering', 'vulnerabilities', 'intrusion detection', 'ZeroImpact, injecting, responsiveness, spoofing, data exfiltration, encrypted, reassembly, reordering, vulnerabilities, intrusion detection', 'ZeroImpact, injecting, responsiveness', 'ZeroImpact, injecting, responsiveness, spoofing, data exfiltration', 'encrypted, reassembly, reordering, vulnerabilities, intrusion detection']\n"
     ]
    }
   ],
   "source": [
    "cos_sim = util.cos_sim(embeddings[0], embeddings)\n",
    "print(cos_sim[0][1:])\n",
    "print(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "id": "2ce56502-4127-4852-9818-f6a236eda480",
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 0\n",
    "\n",
    "sentences = ['Network Packet attack label'] + list(df.loc[i, 'tags'].keys()) + [', '.join(df.loc[i, 'tags'].keys())] + [', '.join(list(df.loc[i, 'tags'].keys())[:3])] + [', '.join(list(df.loc[i, 'tags'].keys())[:5])] + [', '.join(list(df.loc[i, 'tags'].keys())[5:])]\n",
    "\n",
    "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n",
    "embeddings = model.encode(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "id": "5077513e-d882-4b08-9e39-366f8091737d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 0.1084,  0.1272, -0.0045,  0.2424,  0.2889,  0.1280,  0.0351,  0.0356,\n",
      "         0.2897,  0.4265,  0.4104,  0.0445,  0.3287,  0.3273])\n",
      "['Network Packet attack label', 'ZeroImpact', 'injecting', 'responsiveness', 'spoofing', 'data exfiltration', 'encrypted', 'reassembly', 'reordering', 'vulnerabilities', 'intrusion detection', 'ZeroImpact, injecting, responsiveness, spoofing, data exfiltration, encrypted, reassembly, reordering, vulnerabilities, intrusion detection', 'ZeroImpact, injecting, responsiveness', 'ZeroImpact, injecting, responsiveness, spoofing, data exfiltration', 'encrypted, reassembly, reordering, vulnerabilities, intrusion detection']\n"
     ]
    }
   ],
   "source": [
    "cos_sim = util.cos_sim(embeddings[0], embeddings)\n",
    "print(cos_sim[0][1:])\n",
    "print(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "id": "9f99b835-3809-4ae6-a1f8-c031cf7da4bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 0\n",
    "\n",
    "sentences = [test_df['attack_cat'][i]] + list(df.loc[i, 'tags'].keys()) + [', '.join(df.loc[i, 'tags'].keys())] + [','.join(list(df.loc[i, 'tags'].keys())[:3])] + [','.join(list(df.loc[i, 'tags'].keys())[:5])] + [','.join(list(df.loc[i, 'tags'].keys())[6:])]\n",
    "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n",
    "embeddings = model.encode(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "id": "9e8203c3-1aa4-48c0-bfb8-0ac422d9cbc1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.2340, 0.1321, 0.2948, 0.1979, 0.2507, 0.1533, 0.1558, 0.1399, 0.2518,\n",
      "        0.3591, 0.2131, 0.1419, 0.1951, 0.1674])\n",
      "['Analysis', 'ZeroImpact', 'injecting', 'responsiveness', 'spoofing', 'data exfiltration', 'encrypted', 'reassembly', 'reordering', 'vulnerabilities', 'intrusion detection', 'ZeroImpact, injecting, responsiveness, spoofing, data exfiltration, encrypted, reassembly, reordering, vulnerabilities, intrusion detection', 'ZeroImpact,injecting,responsiveness', 'ZeroImpact,injecting,responsiveness,spoofing,data exfiltration', 'reassembly,reordering,vulnerabilities,intrusion detection']\n"
     ]
    }
   ],
   "source": [
    "cos_sim = util.cos_sim(embeddings[0], embeddings)\n",
    "print(cos_sim[0][1:])\n",
    "print(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "id": "b1fa9d71-bc19-4fa4-954f-c36ba0d1e241",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "id": "e4c583de-1673-41f7-849f-56a730f7766e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.0762, 0.0879, 0.0472, 0.2492, 0.3381, 0.0877, 0.0468, 0.0061, 0.3375,\n",
       "        0.4865, 0.4895, 0.0506, 0.3742, 0.4002])"
      ]
     },
     "execution_count": 324,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cos_sim[0][1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "id": "70a0ceba-7685-40be-b46f-ff7f8dcbfea7",
   "metadata": {},
   "outputs": [],
   "source": [
    "similarity_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "878ed257-7f8d-42a0-9475-8cb42dd166e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, packet in enumerate(test_df['attack_cat'].to_list()):\n",
    "    if i > -1:\n",
    "        sentences = [test_df['attack_cat'][i]] + list(df.loc[i, 'tags'].keys()) + [', '.join(df.loc[i, 'tags'].keys())] + [', '.join(list(df.loc[i, 'tags'].keys())[:3])] + [', '.join(list(df.loc[i, 'tags'].keys())[:5])] + [', '.join(list(df.loc[i, 'tags'].keys())[5:])]\n",
    "        embeddings = model.encode(sentences)\n",
    "        cos_sim_attack = util.cos_sim(embeddings[0], embeddings)[0].tolist()\n",
    "        sentences = ['Network Packet'] + list(df.loc[i, 'tags'].keys()) + [', '.join(df.loc[i, 'tags'].keys())] + [', '.join(list(df.loc[i, 'tags'].keys())[:3])] + [', '.join(list(df.loc[i, 'tags'].keys())[:5])] + [', '.join(list(df.loc[i, 'tags'].keys())[5:])]\n",
    "        embeddings = model.encode(sentences)\n",
    "        cos_sim_lingo = util.cos_sim(embeddings[0], embeddings)[0].tolist()\n",
    "        similarity_list.append([(x + y) / 2 for x, y in zip(cos_sim_attack, cos_sim_lingo)])\n",
    "        if i%100 == 0:\n",
    "            print(f'Completed: {i}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "id": "2bb241a2-eb3b-4ab0-8244-4a0512a6cdff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                                                      0\n",
       "tags    {'ZeroImpact': 0.9326767845982844, 'injecting'...\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 320,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "id": "54536534-ae01-449e-a2e8-108a4000dc44",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "packet_dat    9 7 1889 -1 64439 80 240 200 62 0 5 3 -1 71 69...\n",
       "attack_cat                                             Analysis\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 321,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.loc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "id": "767a8b00-bf80-42e7-8fb2-6eef0b1ad25a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46781"
      ]
     },
     "execution_count": 395,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(similarity_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 396,
   "id": "18f05474-66ed-4b1c-84dd-8150561034ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "mdf = pd.DataFrame(similarity_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 397,
   "id": "8ef3e6a2-b663-4d90-8982-1ab73600932c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.165734</td>\n",
       "      <td>0.153081</td>\n",
       "      <td>0.229065</td>\n",
       "      <td>0.203539</td>\n",
       "      <td>0.282942</td>\n",
       "      <td>0.185188</td>\n",
       "      <td>0.105524</td>\n",
       "      <td>0.111780</td>\n",
       "      <td>0.219842</td>\n",
       "      <td>0.357846</td>\n",
       "      <td>0.241838</td>\n",
       "      <td>0.098677</td>\n",
       "      <td>0.227567</td>\n",
       "      <td>0.172046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.212547</td>\n",
       "      <td>0.422899</td>\n",
       "      <td>0.162640</td>\n",
       "      <td>0.274683</td>\n",
       "      <td>0.320445</td>\n",
       "      <td>0.382868</td>\n",
       "      <td>0.104131</td>\n",
       "      <td>0.328420</td>\n",
       "      <td>0.339410</td>\n",
       "      <td>0.097423</td>\n",
       "      <td>0.304525</td>\n",
       "      <td>0.260117</td>\n",
       "      <td>0.304683</td>\n",
       "      <td>0.236394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.123161</td>\n",
       "      <td>0.165813</td>\n",
       "      <td>0.101715</td>\n",
       "      <td>0.149232</td>\n",
       "      <td>0.129282</td>\n",
       "      <td>0.085533</td>\n",
       "      <td>0.190391</td>\n",
       "      <td>0.039290</td>\n",
       "      <td>0.106399</td>\n",
       "      <td>0.113634</td>\n",
       "      <td>0.173009</td>\n",
       "      <td>0.112740</td>\n",
       "      <td>0.106716</td>\n",
       "      <td>0.121531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.174206</td>\n",
       "      <td>0.039290</td>\n",
       "      <td>0.165813</td>\n",
       "      <td>0.107845</td>\n",
       "      <td>0.066654</td>\n",
       "      <td>0.435127</td>\n",
       "      <td>0.111372</td>\n",
       "      <td>0.065605</td>\n",
       "      <td>0.113634</td>\n",
       "      <td>0.149232</td>\n",
       "      <td>0.199035</td>\n",
       "      <td>0.136042</td>\n",
       "      <td>0.167589</td>\n",
       "      <td>0.219220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.165734</td>\n",
       "      <td>0.183315</td>\n",
       "      <td>0.219842</td>\n",
       "      <td>0.213787</td>\n",
       "      <td>0.111780</td>\n",
       "      <td>0.272382</td>\n",
       "      <td>0.185188</td>\n",
       "      <td>0.248124</td>\n",
       "      <td>0.357846</td>\n",
       "      <td>0.041874</td>\n",
       "      <td>0.183145</td>\n",
       "      <td>0.129901</td>\n",
       "      <td>0.162203</td>\n",
       "      <td>0.175423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46776</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.037546</td>\n",
       "      <td>0.206883</td>\n",
       "      <td>0.361385</td>\n",
       "      <td>0.119604</td>\n",
       "      <td>0.166541</td>\n",
       "      <td>0.095780</td>\n",
       "      <td>0.295328</td>\n",
       "      <td>0.320644</td>\n",
       "      <td>0.134524</td>\n",
       "      <td>0.170841</td>\n",
       "      <td>0.250173</td>\n",
       "      <td>0.308410</td>\n",
       "      <td>0.234362</td>\n",
       "      <td>0.222349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46777</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.121958</td>\n",
       "      <td>0.164295</td>\n",
       "      <td>0.183354</td>\n",
       "      <td>0.159643</td>\n",
       "      <td>0.164175</td>\n",
       "      <td>0.153030</td>\n",
       "      <td>0.320644</td>\n",
       "      <td>0.087626</td>\n",
       "      <td>0.137386</td>\n",
       "      <td>0.161356</td>\n",
       "      <td>0.202870</td>\n",
       "      <td>0.161359</td>\n",
       "      <td>0.187015</td>\n",
       "      <td>0.213972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46778</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.259527</td>\n",
       "      <td>0.178403</td>\n",
       "      <td>0.150412</td>\n",
       "      <td>0.222469</td>\n",
       "      <td>0.159309</td>\n",
       "      <td>0.129087</td>\n",
       "      <td>0.325329</td>\n",
       "      <td>0.110329</td>\n",
       "      <td>0.358205</td>\n",
       "      <td>0.046278</td>\n",
       "      <td>0.236216</td>\n",
       "      <td>0.168021</td>\n",
       "      <td>0.198743</td>\n",
       "      <td>0.261288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46779</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.164419</td>\n",
       "      <td>0.370298</td>\n",
       "      <td>0.296461</td>\n",
       "      <td>0.227149</td>\n",
       "      <td>0.095630</td>\n",
       "      <td>0.151565</td>\n",
       "      <td>0.249103</td>\n",
       "      <td>0.498921</td>\n",
       "      <td>0.054749</td>\n",
       "      <td>0.349575</td>\n",
       "      <td>0.262295</td>\n",
       "      <td>0.325645</td>\n",
       "      <td>0.273556</td>\n",
       "      <td>0.285411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46780</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.121958</td>\n",
       "      <td>0.159643</td>\n",
       "      <td>0.153030</td>\n",
       "      <td>0.152809</td>\n",
       "      <td>0.216936</td>\n",
       "      <td>0.125534</td>\n",
       "      <td>0.161356</td>\n",
       "      <td>0.151077</td>\n",
       "      <td>0.339301</td>\n",
       "      <td>0.164295</td>\n",
       "      <td>0.226217</td>\n",
       "      <td>0.167434</td>\n",
       "      <td>0.191890</td>\n",
       "      <td>0.210128</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>46781 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0         1         2         3         4         5         6   \\\n",
       "0      1.0  0.165734  0.153081  0.229065  0.203539  0.282942  0.185188   \n",
       "1      1.0  0.212547  0.422899  0.162640  0.274683  0.320445  0.382868   \n",
       "2      1.0  0.123161  0.165813  0.101715  0.149232  0.129282  0.085533   \n",
       "3      1.0  0.174206  0.039290  0.165813  0.107845  0.066654  0.435127   \n",
       "4      1.0  0.165734  0.183315  0.219842  0.213787  0.111780  0.272382   \n",
       "...    ...       ...       ...       ...       ...       ...       ...   \n",
       "46776  1.0  0.037546  0.206883  0.361385  0.119604  0.166541  0.095780   \n",
       "46777  1.0  0.121958  0.164295  0.183354  0.159643  0.164175  0.153030   \n",
       "46778  1.0  0.259527  0.178403  0.150412  0.222469  0.159309  0.129087   \n",
       "46779  1.0  0.164419  0.370298  0.296461  0.227149  0.095630  0.151565   \n",
       "46780  1.0  0.121958  0.159643  0.153030  0.152809  0.216936  0.125534   \n",
       "\n",
       "             7         8         9         10        11        12        13  \\\n",
       "0      0.105524  0.111780  0.219842  0.357846  0.241838  0.098677  0.227567   \n",
       "1      0.104131  0.328420  0.339410  0.097423  0.304525  0.260117  0.304683   \n",
       "2      0.190391  0.039290  0.106399  0.113634  0.173009  0.112740  0.106716   \n",
       "3      0.111372  0.065605  0.113634  0.149232  0.199035  0.136042  0.167589   \n",
       "4      0.185188  0.248124  0.357846  0.041874  0.183145  0.129901  0.162203   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "46776  0.295328  0.320644  0.134524  0.170841  0.250173  0.308410  0.234362   \n",
       "46777  0.320644  0.087626  0.137386  0.161356  0.202870  0.161359  0.187015   \n",
       "46778  0.325329  0.110329  0.358205  0.046278  0.236216  0.168021  0.198743   \n",
       "46779  0.249103  0.498921  0.054749  0.349575  0.262295  0.325645  0.273556   \n",
       "46780  0.161356  0.151077  0.339301  0.164295  0.226217  0.167434  0.191890   \n",
       "\n",
       "             14  \n",
       "0      0.172046  \n",
       "1      0.236394  \n",
       "2      0.121531  \n",
       "3      0.219220  \n",
       "4      0.175423  \n",
       "...         ...  \n",
       "46776  0.222349  \n",
       "46777  0.213972  \n",
       "46778  0.261288  \n",
       "46779  0.285411  \n",
       "46780  0.210128  \n",
       "\n",
       "[46781 rows x 15 columns]"
      ]
     },
     "execution_count": 397,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 399,
   "id": "a524ddea-3cff-4e2f-984f-0c64bac6e5f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>4.678100e+04</td>\n",
       "      <td>46781.000000</td>\n",
       "      <td>46781.000000</td>\n",
       "      <td>46781.000000</td>\n",
       "      <td>46781.000000</td>\n",
       "      <td>46781.000000</td>\n",
       "      <td>46781.000000</td>\n",
       "      <td>46781.000000</td>\n",
       "      <td>46781.000000</td>\n",
       "      <td>46781.000000</td>\n",
       "      <td>46781.000000</td>\n",
       "      <td>46781.000000</td>\n",
       "      <td>46781.000000</td>\n",
       "      <td>46781.000000</td>\n",
       "      <td>46781.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.219515</td>\n",
       "      <td>0.214031</td>\n",
       "      <td>0.215339</td>\n",
       "      <td>0.215046</td>\n",
       "      <td>0.211866</td>\n",
       "      <td>0.214049</td>\n",
       "      <td>0.211245</td>\n",
       "      <td>0.210009</td>\n",
       "      <td>0.209603</td>\n",
       "      <td>0.208452</td>\n",
       "      <td>0.266279</td>\n",
       "      <td>0.240022</td>\n",
       "      <td>0.258612</td>\n",
       "      <td>0.260090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>7.387071e-08</td>\n",
       "      <td>0.108580</td>\n",
       "      <td>0.106218</td>\n",
       "      <td>0.109244</td>\n",
       "      <td>0.110310</td>\n",
       "      <td>0.112610</td>\n",
       "      <td>0.113846</td>\n",
       "      <td>0.112782</td>\n",
       "      <td>0.112755</td>\n",
       "      <td>0.113025</td>\n",
       "      <td>0.112829</td>\n",
       "      <td>0.086012</td>\n",
       "      <td>0.099156</td>\n",
       "      <td>0.092054</td>\n",
       "      <td>0.090923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>9.999998e-01</td>\n",
       "      <td>-0.076408</td>\n",
       "      <td>-0.076408</td>\n",
       "      <td>-0.076408</td>\n",
       "      <td>-0.076408</td>\n",
       "      <td>-0.076408</td>\n",
       "      <td>-0.076408</td>\n",
       "      <td>-0.076408</td>\n",
       "      <td>-0.076408</td>\n",
       "      <td>-0.076408</td>\n",
       "      <td>-0.056033</td>\n",
       "      <td>0.003943</td>\n",
       "      <td>-0.076668</td>\n",
       "      <td>-0.010201</td>\n",
       "      <td>-0.026272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>9.999999e-01</td>\n",
       "      <td>0.138151</td>\n",
       "      <td>0.137058</td>\n",
       "      <td>0.140748</td>\n",
       "      <td>0.135511</td>\n",
       "      <td>0.132277</td>\n",
       "      <td>0.131804</td>\n",
       "      <td>0.129087</td>\n",
       "      <td>0.126102</td>\n",
       "      <td>0.125510</td>\n",
       "      <td>0.125944</td>\n",
       "      <td>0.211459</td>\n",
       "      <td>0.168650</td>\n",
       "      <td>0.194956</td>\n",
       "      <td>0.198387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.215040</td>\n",
       "      <td>0.211609</td>\n",
       "      <td>0.208698</td>\n",
       "      <td>0.206679</td>\n",
       "      <td>0.198504</td>\n",
       "      <td>0.198504</td>\n",
       "      <td>0.198285</td>\n",
       "      <td>0.197349</td>\n",
       "      <td>0.193769</td>\n",
       "      <td>0.189399</td>\n",
       "      <td>0.264363</td>\n",
       "      <td>0.240741</td>\n",
       "      <td>0.258014</td>\n",
       "      <td>0.258389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.292255</td>\n",
       "      <td>0.281561</td>\n",
       "      <td>0.283656</td>\n",
       "      <td>0.287505</td>\n",
       "      <td>0.281036</td>\n",
       "      <td>0.285785</td>\n",
       "      <td>0.281846</td>\n",
       "      <td>0.280474</td>\n",
       "      <td>0.280592</td>\n",
       "      <td>0.275343</td>\n",
       "      <td>0.324074</td>\n",
       "      <td>0.311717</td>\n",
       "      <td>0.326010</td>\n",
       "      <td>0.322065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.608874</td>\n",
       "      <td>0.639383</td>\n",
       "      <td>0.639383</td>\n",
       "      <td>0.696787</td>\n",
       "      <td>0.696787</td>\n",
       "      <td>0.696787</td>\n",
       "      <td>0.696787</td>\n",
       "      <td>0.696787</td>\n",
       "      <td>0.696787</td>\n",
       "      <td>0.696787</td>\n",
       "      <td>0.550049</td>\n",
       "      <td>0.568608</td>\n",
       "      <td>0.567608</td>\n",
       "      <td>0.594196</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0             1             2             3             4   \\\n",
       "count  4.678100e+04  46781.000000  46781.000000  46781.000000  46781.000000   \n",
       "mean   1.000000e+00      0.219515      0.214031      0.215339      0.215046   \n",
       "std    7.387071e-08      0.108580      0.106218      0.109244      0.110310   \n",
       "min    9.999998e-01     -0.076408     -0.076408     -0.076408     -0.076408   \n",
       "25%    9.999999e-01      0.138151      0.137058      0.140748      0.135511   \n",
       "50%    1.000000e+00      0.215040      0.211609      0.208698      0.206679   \n",
       "75%    1.000000e+00      0.292255      0.281561      0.283656      0.287505   \n",
       "max    1.000000e+00      0.608874      0.639383      0.639383      0.696787   \n",
       "\n",
       "                 5             6             7             8             9   \\\n",
       "count  46781.000000  46781.000000  46781.000000  46781.000000  46781.000000   \n",
       "mean       0.211866      0.214049      0.211245      0.210009      0.209603   \n",
       "std        0.112610      0.113846      0.112782      0.112755      0.113025   \n",
       "min       -0.076408     -0.076408     -0.076408     -0.076408     -0.076408   \n",
       "25%        0.132277      0.131804      0.129087      0.126102      0.125510   \n",
       "50%        0.198504      0.198504      0.198285      0.197349      0.193769   \n",
       "75%        0.281036      0.285785      0.281846      0.280474      0.280592   \n",
       "max        0.696787      0.696787      0.696787      0.696787      0.696787   \n",
       "\n",
       "                 10            11            12            13            14  \n",
       "count  46781.000000  46781.000000  46781.000000  46781.000000  46781.000000  \n",
       "mean       0.208452      0.266279      0.240022      0.258612      0.260090  \n",
       "std        0.112829      0.086012      0.099156      0.092054      0.090923  \n",
       "min       -0.056033      0.003943     -0.076668     -0.010201     -0.026272  \n",
       "25%        0.125944      0.211459      0.168650      0.194956      0.198387  \n",
       "50%        0.189399      0.264363      0.240741      0.258014      0.258389  \n",
       "75%        0.275343      0.324074      0.311717      0.326010      0.322065  \n",
       "max        0.696787      0.550049      0.568608      0.567608      0.594196  "
      ]
     },
     "execution_count": 399,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mdf.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecb7d732-6ff9-43ce-9c17-4a9ba72de1fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# cos_sim_attack = (cos_sim_attack + 1) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a5fc851c-b14d-496e-997a-2906b59fd8ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "variability_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c2b4e6a8-3a1b-4210-a23e-c4ac306d438a",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in df['tags'].to_list():\n",
    "    variability_list.append(list(eval(i).keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "18a52ca8-abfb-4c02-81dd-bb5dbba85d79",
   "metadata": {},
   "outputs": [],
   "source": [
    "tdf = pd.DataFrame(variability_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "760aa4b1-2e9a-4123-b96b-de6d64f7bbbc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ZeroImpact</td>\n",
       "      <td>injecting</td>\n",
       "      <td>responsiveness</td>\n",
       "      <td>spoofing</td>\n",
       "      <td>data exfiltration</td>\n",
       "      <td>encrypted</td>\n",
       "      <td>reassembly</td>\n",
       "      <td>reordering</td>\n",
       "      <td>vulnerabilities</td>\n",
       "      <td>intrusion detection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>injecting unauthorized packets</td>\n",
       "      <td>packet injection</td>\n",
       "      <td>exploit vulnerabilities</td>\n",
       "      <td>injects malicious packets</td>\n",
       "      <td>SYN packets overwhelming</td>\n",
       "      <td>HTTP packets</td>\n",
       "      <td>Host Header Injection attack</td>\n",
       "      <td>inject packets</td>\n",
       "      <td>attacker intercepts network traffic</td>\n",
       "      <td>buffer overflow input validation issues</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unknowingly triggering</td>\n",
       "      <td>exploiting vulnerabilities</td>\n",
       "      <td>VulneraKit</td>\n",
       "      <td>compromising</td>\n",
       "      <td>ExploitBot attack</td>\n",
       "      <td>ExploShock attack typically</td>\n",
       "      <td>ExploitAnalyzer</td>\n",
       "      <td>mitigate ExploDos</td>\n",
       "      <td>misconfigured</td>\n",
       "      <td>ReconFuzz</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Exploit DoS</td>\n",
       "      <td>mitigate ExploDos</td>\n",
       "      <td>exploiting vulnerabilities</td>\n",
       "      <td>mitigate ScanFuzz attacks</td>\n",
       "      <td>Exploit Normalizer</td>\n",
       "      <td>TCP</td>\n",
       "      <td>misconfigurations</td>\n",
       "      <td>ReconFuseDoS</td>\n",
       "      <td>ReconFuzz</td>\n",
       "      <td>compromising</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ZeroImpact</td>\n",
       "      <td>anomalies</td>\n",
       "      <td>vulnerabilities</td>\n",
       "      <td>intricacies</td>\n",
       "      <td>reordering</td>\n",
       "      <td>DataGatheringGrenade</td>\n",
       "      <td>encrypted</td>\n",
       "      <td>evade detection</td>\n",
       "      <td>intrusion detection</td>\n",
       "      <td>prevent SQLStorm attacks</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46776</th>\n",
       "      <td>successful Exploit Normalizer attack</td>\n",
       "      <td>malicious actor exploits vulnerabilities</td>\n",
       "      <td>attacker sends specially crafted network packets</td>\n",
       "      <td>detecting malformed</td>\n",
       "      <td>malicious activities including remote code exe...</td>\n",
       "      <td>filter FTP</td>\n",
       "      <td>malicious traffic diverting legitimate</td>\n",
       "      <td>Sockets Layer</td>\n",
       "      <td>clients updated</td>\n",
       "      <td>Vulnerability</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46777</th>\n",
       "      <td>misconfigured firewalls weak authentication me...</td>\n",
       "      <td>Vulnerability Scanner</td>\n",
       "      <td>disguising</td>\n",
       "      <td>unauthorized</td>\n",
       "      <td>ExploitSQL</td>\n",
       "      <td>PersistentIntrusion attack</td>\n",
       "      <td>Sockets Layer</td>\n",
       "      <td>VulnerabilityChain attack relies</td>\n",
       "      <td>exploiting multiple vulnerabilities</td>\n",
       "      <td>exploits vulnerabilities</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46778</th>\n",
       "      <td>malicious code</td>\n",
       "      <td>infectious exploits involves regular patching</td>\n",
       "      <td>specific vulnerabilities discovered allowing</td>\n",
       "      <td>detecting infectious exploits</td>\n",
       "      <td>exploit vulnerabilities</td>\n",
       "      <td>sensitive information</td>\n",
       "      <td>malicious activity</td>\n",
       "      <td>execute arbitrary commands install malware</td>\n",
       "      <td>Additionally network traffic monitoring intrusion</td>\n",
       "      <td>software bugs configuration weaknesses</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46779</th>\n",
       "      <td>Vulnerability Hunter</td>\n",
       "      <td>injects malicious packets</td>\n",
       "      <td>attacks network administrators</td>\n",
       "      <td>security measures including encryption protoco...</td>\n",
       "      <td>unauthorized location</td>\n",
       "      <td>simple buffer overflow attacks</td>\n",
       "      <td>employing robust network security measures</td>\n",
       "      <td>network packet attack</td>\n",
       "      <td>diverting sensitive</td>\n",
       "      <td>Additionally monitoring network traffic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46780</th>\n",
       "      <td>misconfigured firewalls weak authentication me...</td>\n",
       "      <td>unauthorized</td>\n",
       "      <td>PersistentIntrusion attack</td>\n",
       "      <td>Malware Launcher</td>\n",
       "      <td>StealthySilencer</td>\n",
       "      <td>EncryptionBreach</td>\n",
       "      <td>exploits vulnerabilities</td>\n",
       "      <td>VulnerabilityChain</td>\n",
       "      <td>Packet spoofing</td>\n",
       "      <td>Vulnerability Scanner</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>46781 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                       0  \\\n",
       "0                                             ZeroImpact   \n",
       "1                         injecting unauthorized packets   \n",
       "2                                 unknowingly triggering   \n",
       "3                                            Exploit DoS   \n",
       "4                                             ZeroImpact   \n",
       "...                                                  ...   \n",
       "46776               successful Exploit Normalizer attack   \n",
       "46777  misconfigured firewalls weak authentication me...   \n",
       "46778                                     malicious code   \n",
       "46779                               Vulnerability Hunter   \n",
       "46780  misconfigured firewalls weak authentication me...   \n",
       "\n",
       "                                                   1  \\\n",
       "0                                          injecting   \n",
       "1                                   packet injection   \n",
       "2                         exploiting vulnerabilities   \n",
       "3                                  mitigate ExploDos   \n",
       "4                                          anomalies   \n",
       "...                                              ...   \n",
       "46776       malicious actor exploits vulnerabilities   \n",
       "46777                          Vulnerability Scanner   \n",
       "46778  infectious exploits involves regular patching   \n",
       "46779                      injects malicious packets   \n",
       "46780                                   unauthorized   \n",
       "\n",
       "                                                      2  \\\n",
       "0                                        responsiveness   \n",
       "1                               exploit vulnerabilities   \n",
       "2                                            VulneraKit   \n",
       "3                            exploiting vulnerabilities   \n",
       "4                                       vulnerabilities   \n",
       "...                                                 ...   \n",
       "46776  attacker sends specially crafted network packets   \n",
       "46777                                        disguising   \n",
       "46778      specific vulnerabilities discovered allowing   \n",
       "46779                    attacks network administrators   \n",
       "46780                        PersistentIntrusion attack   \n",
       "\n",
       "                                                       3  \\\n",
       "0                                               spoofing   \n",
       "1                              injects malicious packets   \n",
       "2                                           compromising   \n",
       "3                              mitigate ScanFuzz attacks   \n",
       "4                                            intricacies   \n",
       "...                                                  ...   \n",
       "46776                                detecting malformed   \n",
       "46777                                       unauthorized   \n",
       "46778                      detecting infectious exploits   \n",
       "46779  security measures including encryption protoco...   \n",
       "46780                                   Malware Launcher   \n",
       "\n",
       "                                                       4  \\\n",
       "0                                      data exfiltration   \n",
       "1                               SYN packets overwhelming   \n",
       "2                                      ExploitBot attack   \n",
       "3                                     Exploit Normalizer   \n",
       "4                                             reordering   \n",
       "...                                                  ...   \n",
       "46776  malicious activities including remote code exe...   \n",
       "46777                                         ExploitSQL   \n",
       "46778                            exploit vulnerabilities   \n",
       "46779                              unauthorized location   \n",
       "46780                                   StealthySilencer   \n",
       "\n",
       "                                    5  \\\n",
       "0                           encrypted   \n",
       "1                        HTTP packets   \n",
       "2         ExploShock attack typically   \n",
       "3                                 TCP   \n",
       "4                DataGatheringGrenade   \n",
       "...                               ...   \n",
       "46776                      filter FTP   \n",
       "46777      PersistentIntrusion attack   \n",
       "46778           sensitive information   \n",
       "46779  simple buffer overflow attacks   \n",
       "46780                EncryptionBreach   \n",
       "\n",
       "                                                6  \\\n",
       "0                                      reassembly   \n",
       "1                    Host Header Injection attack   \n",
       "2                                 ExploitAnalyzer   \n",
       "3                               misconfigurations   \n",
       "4                                       encrypted   \n",
       "...                                           ...   \n",
       "46776      malicious traffic diverting legitimate   \n",
       "46777                               Sockets Layer   \n",
       "46778                          malicious activity   \n",
       "46779  employing robust network security measures   \n",
       "46780                    exploits vulnerabilities   \n",
       "\n",
       "                                                7  \\\n",
       "0                                      reordering   \n",
       "1                                  inject packets   \n",
       "2                               mitigate ExploDos   \n",
       "3                                    ReconFuseDoS   \n",
       "4                                 evade detection   \n",
       "...                                           ...   \n",
       "46776                               Sockets Layer   \n",
       "46777            VulnerabilityChain attack relies   \n",
       "46778  execute arbitrary commands install malware   \n",
       "46779                       network packet attack   \n",
       "46780                          VulnerabilityChain   \n",
       "\n",
       "                                                       8  \\\n",
       "0                                        vulnerabilities   \n",
       "1                    attacker intercepts network traffic   \n",
       "2                                          misconfigured   \n",
       "3                                              ReconFuzz   \n",
       "4                                    intrusion detection   \n",
       "...                                                  ...   \n",
       "46776                                    clients updated   \n",
       "46777                exploiting multiple vulnerabilities   \n",
       "46778  Additionally network traffic monitoring intrusion   \n",
       "46779                                diverting sensitive   \n",
       "46780                                    Packet spoofing   \n",
       "\n",
       "                                             9  \n",
       "0                          intrusion detection  \n",
       "1      buffer overflow input validation issues  \n",
       "2                                    ReconFuzz  \n",
       "3                                 compromising  \n",
       "4                     prevent SQLStorm attacks  \n",
       "...                                        ...  \n",
       "46776                            Vulnerability  \n",
       "46777                 exploits vulnerabilities  \n",
       "46778   software bugs configuration weaknesses  \n",
       "46779  Additionally monitoring network traffic  \n",
       "46780                    Vulnerability Scanner  \n",
       "\n",
       "[46781 rows x 10 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "3165b1d6-b98f-4e1c-a366-29af9fcf15c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "tdf['c'] = tdf.iloc[:, 0:10].apply(lambda x: ' '.join(x.astype(str)), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "e4f5e711-e95b-4d7d-9a8b-20c7bb970d19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ZeroImpact injecting responsiveness spoofing data exfiltration encrypted reassembly reordering vulnerabilities intrusion detection'"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tdf['c'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "4ad5eda7-ce8a-495e-9524-c025e06581ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42766"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tdf['c'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "93284276-f50a-411b-bee6-93cb079129f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2302\n",
       "1    3051\n",
       "2    3712\n",
       "3    4249\n",
       "4    4685\n",
       "5    5214\n",
       "6    5631\n",
       "7    6092\n",
       "8    6424\n",
       "9    6036\n",
       "dtype: int64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf8646f9-114f-4638-9479-48db3b1eedd6",
   "metadata": {},
   "source": [
    "# Text for Tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b60643a5-e027-4b5c-b2c1-e59bf8b09293",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3293839b-7da7-4eb2-9bee-285eebc243bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./GENERATED_TAGS.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c7abb564-064f-4685-a75e-f7c25c35d8fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>{'ZeroImpact': 0.9326767845982844, 'injecting'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>{'injecting unauthorized packets': 0.929202050...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>{'unknowingly triggering': 0.9352225230817839,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>{'Exploit DoS': 0.9215557725519854, 'mitigate ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>{'ZeroImpact': 0.9545252141232174, 'anomalies'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46776</th>\n",
       "      <td>46776</td>\n",
       "      <td>{'successful Exploit Normalizer attack': 0.940...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46777</th>\n",
       "      <td>46777</td>\n",
       "      <td>{'misconfigured firewalls weak authentication ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46778</th>\n",
       "      <td>46778</td>\n",
       "      <td>{'malicious code': 0.9816162270729385, 'infect...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46779</th>\n",
       "      <td>46779</td>\n",
       "      <td>{'Vulnerability Hunter': 0.9274233934166645, '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46780</th>\n",
       "      <td>46780</td>\n",
       "      <td>{'misconfigured firewalls weak authentication ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>46781 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                               tags\n",
       "0          0  {'ZeroImpact': 0.9326767845982844, 'injecting'...\n",
       "1          1  {'injecting unauthorized packets': 0.929202050...\n",
       "2          2  {'unknowingly triggering': 0.9352225230817839,...\n",
       "3          3  {'Exploit DoS': 0.9215557725519854, 'mitigate ...\n",
       "4          4  {'ZeroImpact': 0.9545252141232174, 'anomalies'...\n",
       "...      ...                                                ...\n",
       "46776  46776  {'successful Exploit Normalizer attack': 0.940...\n",
       "46777  46777  {'misconfigured firewalls weak authentication ...\n",
       "46778  46778  {'malicious code': 0.9816162270729385, 'infect...\n",
       "46779  46779  {'Vulnerability Hunter': 0.9274233934166645, '...\n",
       "46780  46780  {'misconfigured firewalls weak authentication ...\n",
       "\n",
       "[46781 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dae3c791-28e4-4fd4-b44c-6efb10c37d5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"{'ZeroImpact': 0.9326767845982844, 'injecting': 0.9281784060320444, 'responsiveness': 0.9276119540057473, 'spoofing': 0.9275524398471746, 'data exfiltration': 0.9271565682528817, 'encrypted': 0.9271299868504239, 'reassembly': 0.9270286951368641, 'reordering': 0.9263218957220968, 'vulnerabilities': 0.9262331496090768, 'intrusion detection': 0.9258145334115236}\""
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['tags'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "76637983-f6fa-433e-af98-5ef8cae42397",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['tags'] = df['tags'].apply(lambda x: [key for key in eval(x).keys()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "784ffe2c-501a-4f3a-b2ec-54fc86ad1814",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ZeroImpact',\n",
       " 'reordering',\n",
       " 'anomalies',\n",
       " 'vulnerabilities',\n",
       " 'DataGatheringGrenade',\n",
       " 'RegularOperation',\n",
       " 'firewalls intrusion',\n",
       " 'intricacies',\n",
       " 'reroute',\n",
       " 'StealthySilencer']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['tags'][516]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
